00000800:00000100:3.1F:1788877118.521046:0:2353:0:(socklnd_cb.c:2294:ksocknal_find_timed_out_conn()) Timeout sending data to 12345-192.168.203.129@tcp (192.168.203.129:988) the network or that node may be down. 00000100:00080000:2.0:1788878079.481272:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878079.481273:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878079.481275:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878079.481277:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878079.481280:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9018 00000100:00080000:2.0:1788878079.481283:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878079.481284:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878079.481285:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878079.481288:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878079.481289:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878079.481298:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878079.481304:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878079.481308:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878079.481310:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878079.481315:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878079.481321:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878079.481325:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878079.481331:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878079.481336:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878079.481342:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878079.481346:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878079.481351:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878079.481352:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878079.481357:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878079.481358:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878079.481363:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878079.481364:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878079.481366:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878079.481370:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878079.481373:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290027648/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878079.481381:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878079.481384:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878079.481398:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878079.481402:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878079.481411:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878079.481414:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878079.481417:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878079.481421:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878079.481440:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9018 00000100:00080000:2.0:1788878079.481446:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878079.481451:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878079.481454:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878079.481459:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878079.481461:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878079.481464:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878079.481466:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878079.481469:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878079.481470:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878079.481477:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878079.481481:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878079.481485:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878079.481489:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878079.481494:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878079.481501:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878079.481502:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878079.481508:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878079.481512:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878079.481518:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878079.481521:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878079.481524:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878079.481527:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878079.481531:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878079.481537:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878079.481547:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290027776/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878079.481558:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878079.481563:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878079.481568:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878084.600974:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878084.600982:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878084.600985:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878084.600988:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878084.600992:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878084.600999:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9023 00000100:00080000:2.0:1788878084.601009:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878084.601014:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878084.601016:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878084.601024:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878084.601026:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878084.601029:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878084.601033:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878084.601036:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878084.601038:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878084.601043:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878084.601046:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878084.601050:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878084.601053:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878084.601056:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601058:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878084.601060:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878084.601068:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878084.601070:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601072:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878084.601075:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878084.601079:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601089:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878084.601099:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290027904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878084.601111:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878084.601116:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601130:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878084.601134:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878084.601138:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878084.601140:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878084.601141:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878084.601144:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878084.601148:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9023 00000100:00080000:2.0:1788878084.601153:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878084.601156:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878084.601158:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878084.601162:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878084.601164:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878084.601166:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878084.601169:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878084.601171:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878084.601173:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878084.601176:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878084.601180:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878084.601182:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878084.601188:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878084.601190:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601193:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878084.601198:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878084.601202:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601204:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878084.601208:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878084.601210:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601212:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878084.601214:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878084.601216:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601223:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878084.601229:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290028032/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878084.601237:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878084.601239:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601242:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878084.601245:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878084.601249:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878084.601251:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878084.601252:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878084.601255:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878084.601258:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9023 00000100:00080000:2.0:1788878084.601262:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878084.601265:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878084.601266:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878084.601270:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878084.601272:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878084.601274:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878084.601276:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878084.601279:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878084.601280:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878084.601283:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878084.601286:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878084.601288:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878084.601291:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878084.601293:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601295:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878084.601298:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878084.601301:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601303:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878084.601307:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878084.601309:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601311:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878084.601312:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878084.601314:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601319:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878084.601324:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290028160/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878084.601332:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878084.601334:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601336:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878084.601339:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878084.601343:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878084.601345:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878084.601346:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878084.601350:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878084.601352:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9023 00000100:00080000:2.0:1788878084.601355:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878084.601358:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878084.601359:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878084.601362:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878084.601364:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878084.601366:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878084.601369:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878084.601371:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878084.601373:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878084.601376:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878084.601379:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878084.601381:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878084.601383:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878084.601385:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601388:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878084.601389:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878084.601392:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601394:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878084.601398:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878084.601400:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601416:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878084.601418:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878084.601420:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601425:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878084.601430:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290028288/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878084.601439:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878084.601442:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878084.601446:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878099.961082:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878099.961104:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878099.961107:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878099.961111:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878099.961115:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878099.961122:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9038 00000100:00080000:2.0:1788878099.961132:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878099.961136:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878099.961138:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878099.961146:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878099.961149:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878099.961152:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878099.961155:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878099.961158:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878099.961159:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878099.961165:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878099.961168:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878099.961170:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878099.961174:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878099.961177:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961178:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878099.961180:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878099.961191:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878099.961192:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961194:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878099.961196:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878099.961200:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961210:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878099.961217:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290029440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878099.961225:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878099.961228:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961246:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878099.961249:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878099.961251:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878099.961252:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878099.961254:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878099.961256:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878099.961258:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9038 00000100:00080000:2.0:1788878099.961260:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878099.961262:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878099.961263:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878099.961265:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878099.961266:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878099.961268:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878099.961269:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878099.961271:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878099.961272:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878099.961274:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878099.961276:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878099.961277:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878099.961282:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878099.961283:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961284:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878099.961288:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878099.961291:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961293:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878099.961296:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878099.961297:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961299:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878099.961300:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878099.961301:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961305:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878099.961310:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290029568/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878099.961316:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878099.961318:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961320:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878099.961322:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878099.961324:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878099.961325:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878099.961326:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878099.961327:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878099.961332:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9038 00000100:00080000:2.0:1788878099.961351:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878099.961352:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878099.961353:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878099.961356:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878099.961359:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878099.961367:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878099.961378:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878099.961386:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878099.961394:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878099.961412:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878099.961420:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878099.961425:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878099.961433:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878099.961439:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961446:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878099.961455:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878099.961463:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961466:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878099.961473:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878099.961476:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961479:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878099.961486:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878099.961488:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961498:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878099.961515:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290029696/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878099.961520:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878099.961522:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961526:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878099.961528:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878099.961531:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878099.961532:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878099.961533:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878099.961535:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878099.961539:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9038 00000100:00080000:2.0:1788878099.961549:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878099.961557:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878099.961559:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878099.961563:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878099.961570:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878099.961574:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878099.961576:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878099.961578:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878099.961579:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878099.961581:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878099.961583:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878099.961584:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878099.961586:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878099.961588:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961589:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878099.961590:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878099.961592:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961593:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878099.961596:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878099.961597:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961598:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878099.961599:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878099.961600:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961603:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878099.961606:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290029824/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878099.961612:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878099.961613:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878099.961615:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878105.081056:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878105.081068:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878105.081071:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878105.081076:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878105.081099:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878105.081108:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9043 00000100:00080000:2.0:1788878105.081122:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878105.081126:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878105.081129:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878105.081139:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878105.081142:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878105.081147:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878105.081150:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878105.081152:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878105.081154:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878105.081160:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878105.081163:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878105.081166:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878105.081170:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878105.081174:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.081177:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878105.081180:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878105.081193:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878105.081195:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.081199:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878105.081202:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878105.081208:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.081222:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878105.081234:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5f80 x1875765290029952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878105.081248:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878105.081254:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.081282:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878105.081287:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878105.081292:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878105.081294:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878105.081297:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878105.081300:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878105.081304:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9043 00000100:00080000:2.0:1788878105.081308:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878105.081353:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878105.081356:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878105.081363:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878105.081366:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878105.081370:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878105.081374:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878105.081380:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878105.081383:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878105.081391:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878105.081396:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878105.081416:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878105.081434:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878105.081441:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.081447:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878105.081458:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878105.081483:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.081489:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878105.081496:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878105.081504:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.081511:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878105.081519:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878105.081529:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.081548:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878105.081566:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4380 x1875765290030080/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878105.081594:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878105.081602:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.081611:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878105.081616:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878105.081624:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878105.081627:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878105.081629:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878105.081634:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878105.081645:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9043 00000100:00080000:2.0:1788878105.081655:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878105.081659:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878105.081662:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878105.081672:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878105.081679:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878105.081684:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878105.081693:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878105.081697:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878105.081703:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878105.081711:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878105.081716:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878105.081719:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878105.081724:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878105.081730:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.081733:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878105.081736:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878105.081739:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.081741:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878105.081749:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878105.081754:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.081758:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878105.081760:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878105.081764:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.081779:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878105.081791:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290030208/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878105.081821:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878105.081829:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.081837:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878105.081845:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878105.081851:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878105.081854:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878105.081861:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878105.081868:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878105.081873:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9043 00000100:00080000:2.0:1788878105.081880:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878105.081886:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878105.081889:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878105.081901:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878105.081903:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878105.081906:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878105.081908:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878105.081911:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878105.081913:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878105.081964:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878105.081973:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878105.081981:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878105.081988:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878105.081995:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.082001:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878105.082006:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878105.082014:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.082021:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878105.082031:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878105.082037:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.082045:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878105.082052:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878105.082059:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.082070:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878105.082087:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290030336/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878105.082107:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878105.082113:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878105.082120:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.201386:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878110.201398:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878110.201407:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878110.201414:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878110.201419:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878110.201430:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878110.201433:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878110.201443:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878110.201453:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290030464:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878110.201464:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878110.201471:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878110.201475:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878110.201479:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878110.201488:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.201508:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878110.201514:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286519680 : -117763423031936 : ffff94e51100ad80) 00000100:00000001:2.0:1788878110.201523:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878110.201580:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878110.201603:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878110.201612:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878110.201621:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.201628:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878110.201638:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.201654:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303722025 00000100:00000001:2.0:1788878110.201673:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878110.201688:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303722025 00000100:00000001:2.0:1788878110.201709:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878110.201721:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878110.201736:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.201744:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878110.201762:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878110/real 1788878110] req@ffff94e51100ad80 x1875765290030464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878110.201815:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878110.201827:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878110.201837:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878110.201844:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878110.201846:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.201861:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878110.201864:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878110.201874:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100ad80 x1875765290030464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878110.201884:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.201890:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878110.201894:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.201895:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878110.201897:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.201899:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878110.201939:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878110.201943:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878110.201948:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878110.201951:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878110.201956:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290030464:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878110.201961:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.201963:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878110.201964:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878110.201967:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878110.201971:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878110.201975:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878110.201977:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878110.201982:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878110.201984:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878110.201987:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878110.201989:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.201991:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.201998:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878110.202001:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.202003:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878110.202005:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878110.202006:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878110.202008:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878110.202010:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878110.202013:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878110.202015:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878110.202018:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878110.202022:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290030592:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878110.202036:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878110.202039:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878110.202041:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878110.202043:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878110.202046:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.202052:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878110.202054:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508032 : -117763423043584 : ffff94e511008000) 00000100:00000001:2.0:1788878110.202058:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878110.202075:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878110.202079:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878110.202082:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878110.202083:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.202085:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878110.202088:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.202102:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303721484 00000100:00000001:2.0:1788878110.202105:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303722025 00000100:00000001:2.0:1788878110.202108:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303721484 00000100:00000001:2.0:1788878110.202110:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303722025 00000100:00000001:2.0:1788878110.202113:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878110.202115:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.202117:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878110.202121:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878110/real 1788878110] req@ffff94e511008000 x1875765290030592/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878110.202131:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878110.202132:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878110.202135:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878110.202137:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878110.202139:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202141:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878110.202142:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878110.202146:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008000 x1875765290030592/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878110.202155:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202158:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878110.202161:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202163:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878110.202164:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202166:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878110.202180:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878110.202184:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878110.202188:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878110.202190:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878110.202194:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290030592:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878110.202198:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878110.202200:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878110.202201:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878110.202204:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878110.202206:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878110.202208:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878110.202212:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290030720:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878110.202215:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878110.202218:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878110.202219:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878110.202221:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878110.202224:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.202231:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878110.202232:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286511616 : -117763423040000 : ffff94e511008e00) 00000100:00000001:2.0:1788878110.202237:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878110.202251:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878110.202255:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878110.202257:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878110.202258:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.202260:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878110.202263:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.202266:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303721482 00000100:00000001:2.0:1788878110.202269:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303722025 00000100:00000001:2.0:1788878110.202272:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303721482 00000100:00000001:2.0:1788878110.202274:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303722025 00000100:00000001:2.0:1788878110.202276:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878110.202278:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.202280:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878110.202285:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878110/real 1788878110] req@ffff94e511008e00 x1875765290030720/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878110.202312:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878110.202321:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878110.202324:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878110.202326:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878110.202328:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202330:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878110.202331:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878110.202336:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008e00 x1875765290030720/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878110.202350:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202353:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878110.202355:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202357:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878110.202359:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202361:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878110.202373:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878110.202376:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878110.202380:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878110.202382:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878110.202386:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290030720:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878110.202390:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878110.202392:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878110.202394:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878110.202396:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878110.202398:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878110.202401:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878110.202404:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290030848:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878110.202408:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878110.202410:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878110.202412:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878110.202414:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878110.202417:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.202422:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878110.202424:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286509824 : -117763423041792 : ffff94e511008700) 00000100:00000001:2.0:1788878110.202427:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878110.202439:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878110.202444:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878110.202446:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878110.202447:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.202450:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878110.202452:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.202456:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303721482 00000100:00000001:2.0:1788878110.202458:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303722025 00000100:00000001:2.0:1788878110.202461:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303721482 00000100:00000001:2.0:1788878110.202463:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303722025 00000100:00000001:2.0:1788878110.202465:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878110.202468:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.202469:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878110.202474:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878110/real 1788878110] req@ffff94e511008700 x1875765290030848/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878110.202482:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878110.202484:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878110.202486:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878110.202488:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878110.202490:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202492:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878110.202493:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878110.202497:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008700 x1875765290030848/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878110.202504:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202506:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878110.202509:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202511:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878110.202512:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202514:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878110.202525:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878110.202528:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878110.202532:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878110.202534:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878110.202538:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290030848:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878110.202542:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202544:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878110.202545:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878110.202547:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878110.202549:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878110.202551:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878110.202553:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878110.202555:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878110.202556:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878110.202558:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878110.202560:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202562:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878110.202563:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878110.202565:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878110.202566:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878110.202568:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878110.202569:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878110.202571:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878110.202572:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878110.202574:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878110.202575:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202577:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878110.202578:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878110.202579:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878110.202581:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878110.202582:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878110.202583:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878110.202585:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878110.202587:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878110.202589:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878110.202590:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202592:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878110.202595:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878110.202597:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878110.202599:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878110.202600:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.321286:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878115.321294:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878115.321299:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878115.321307:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878115.321312:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878115.321319:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878115.321325:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878115.321330:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878115.321337:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290030976:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878115.321345:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878115.321353:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878115.321357:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878115.321361:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878115.321369:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.321391:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878115.321396:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:2.0:1788878115.321403:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878115.321461:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878115.321490:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878115.321499:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878115.321506:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.321510:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878115.321519:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.321526:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303727145 00000100:00000001:2.0:1788878115.321544:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878115.321564:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303727145 00000100:00000001:2.0:1788878115.321575:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878115.321584:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878115.321591:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.321596:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878115.321610:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878115/real 1788878115] req@ffff94e50449d180 x1875765290030976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878115.321640:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878115.321646:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878115.321655:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878115.321660:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878115.321666:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.321674:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878115.321678:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878115.321690:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290030976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878115.321730:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.321737:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878115.321744:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.321750:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878115.321753:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.321758:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878115.321763:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878115.321773:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878115.321786:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878115.321791:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878115.321801:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290030976:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878115.321814:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.321818:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878115.321824:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878115.321831:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878115.321836:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878115.321840:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878115.321843:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878115.321847:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878115.321849:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878115.321852:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878115.321853:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.321856:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.321863:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878115.321865:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.321869:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878115.321871:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878115.321873:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878115.321875:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878115.321877:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878115.321880:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878115.321883:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878115.321886:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878115.321889:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290031104:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878115.321893:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878115.321895:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878115.321896:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878115.321898:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878115.321935:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.321942:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878115.321944:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:2.0:1788878115.321949:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878115.321962:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878115.321967:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878115.321969:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878115.321971:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.321973:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878115.321976:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.321979:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303726604 00000100:00000001:2.0:1788878115.321983:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303727145 00000100:00000001:2.0:1788878115.321985:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303726604 00000100:00000001:2.0:1788878115.321988:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303727145 00000100:00000001:2.0:1788878115.321990:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878115.321992:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.321994:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878115.321998:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878115/real 1788878115] req@ffff94e50449c000 x1875765290031104/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878115.322006:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878115.322008:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878115.322010:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878115.322012:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878115.322013:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322015:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878115.322016:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878115.322021:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290031104/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878115.322028:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322031:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878115.322033:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322034:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878115.322036:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322037:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878115.322049:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878115.322052:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878115.322055:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878115.322057:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878115.322060:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290031104:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878115.322063:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878115.322065:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878115.322067:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878115.322069:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878115.322071:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878115.322073:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878115.322076:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290031232:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878115.322078:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878115.322097:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878115.322100:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878115.322102:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878115.322106:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.322112:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878115.322114:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:2.0:1788878115.322119:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878115.322132:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878115.322137:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878115.322140:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878115.322142:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.322145:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878115.322165:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.322169:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303726602 00000100:00000001:2.0:1788878115.322171:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303727145 00000100:00000001:2.0:1788878115.322174:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303726602 00000100:00000001:2.0:1788878115.322177:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303727145 00000100:00000001:2.0:1788878115.322179:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878115.322181:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.322183:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878115.322188:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878115/real 1788878115] req@ffff94e50449f480 x1875765290031232/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878115.322196:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878115.322198:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878115.322201:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878115.322202:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878115.322204:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322206:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878115.322208:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878115.322212:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290031232/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878115.322219:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322222:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878115.322224:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322226:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878115.322227:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322229:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878115.322240:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878115.322243:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878115.322246:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878115.322248:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878115.322252:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290031232:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878115.322257:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878115.322258:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878115.322260:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878115.322263:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878115.322302:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878115.322304:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878115.322307:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290031360:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878115.322310:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878115.322312:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878115.322314:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878115.322316:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878115.322319:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.322324:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878115.322326:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:2.0:1788878115.322330:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878115.322341:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878115.322346:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878115.322348:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878115.322350:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.322351:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878115.322354:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.322357:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303726602 00000100:00000001:2.0:1788878115.322359:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303727145 00000100:00000001:2.0:1788878115.322362:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303726602 00000100:00000001:2.0:1788878115.322364:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303727145 00000100:00000001:2.0:1788878115.322366:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878115.322368:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.322370:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878115.322373:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878115/real 1788878115] req@ffff94e50449c700 x1875765290031360/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878115.322388:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878115.322389:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878115.322392:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878115.322393:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878115.322394:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322396:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878115.322398:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878115.322401:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290031360/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878115.322408:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322410:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878115.322412:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322414:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878115.322415:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322417:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878115.322427:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878115.322430:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878115.322432:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878115.322434:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878115.322437:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290031360:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878115.322440:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322442:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878115.322443:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878115.322445:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878115.322447:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878115.322449:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878115.322450:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878115.322453:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878115.322454:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878115.322456:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878115.322458:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322460:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878115.322461:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878115.322462:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878115.322465:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878115.322467:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878115.322468:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878115.322469:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878115.322470:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878115.322472:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878115.322473:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322475:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878115.322476:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878115.322477:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878115.322478:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878115.322479:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878115.322480:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878115.322482:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878115.322483:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878115.322484:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878115.322485:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322487:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878115.322490:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878115.322491:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878115.322493:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878115.322495:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.441424:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878120.441434:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878120.441440:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878120.441446:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878120.441451:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878120.441464:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878120.441482:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878120.441491:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878120.441508:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290031488:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878120.441522:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878120.441529:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878120.441534:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878120.441540:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878120.441560:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.441585:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878120.441592:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:2.0:1788878120.441608:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878120.441665:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878120.441682:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878120.441691:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878120.441697:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.441706:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878120.441727:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.441735:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303732265 00000100:00000001:2.0:1788878120.441746:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878120.441758:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303732265 00000100:00000001:2.0:1788878120.441768:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878120.441790:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878120.441797:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.441806:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878120.441827:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878120/real 1788878120] req@ffff94e50897ad80 x1875765290031488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878120.441852:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878120.441859:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878120.441870:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878120.441877:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878120.441886:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.441890:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878120.441899:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878120.441954:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290031488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878120.441962:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.441970:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878120.441974:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.441979:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878120.441983:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.441988:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878120.441995:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878120.442021:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878120.442034:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878120.442042:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878120.442055:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290031488:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878120.442089:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.442098:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878120.442109:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878120.442127:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878120.442134:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878120.442159:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878120.442168:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878120.442178:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878120.442194:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878120.442205:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878120.442208:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.442213:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.442222:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878120.442228:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.442257:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878120.442265:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878120.442272:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878120.442305:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878120.442308:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878120.442315:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878120.442319:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878120.442325:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878120.442333:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290031616:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878120.442340:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878120.442360:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878120.442361:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878120.442363:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878120.442366:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.442373:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878120.442374:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:2.0:1788878120.442378:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878120.442394:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878120.442399:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878120.442401:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878120.442402:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.442404:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878120.442406:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.442410:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303731724 00000100:00000001:2.0:1788878120.442413:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303732265 00000100:00000001:2.0:1788878120.442416:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303731724 00000100:00000001:2.0:1788878120.442418:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303732265 00000100:00000001:2.0:1788878120.442433:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878120.442435:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.442454:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878120.442462:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878120/real 1788878120] req@ffff94e50897bb80 x1875765290031616/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878120.442476:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878120.442478:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878120.442484:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878120.442487:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878120.442489:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.442493:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878120.442495:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878120.442502:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290031616/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878120.442528:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.442531:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878120.442534:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.442536:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878120.442537:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.442539:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878120.442556:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878120.442560:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878120.442566:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878120.442569:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878120.442573:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290031616:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878120.442578:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878120.442580:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878120.442582:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878120.442585:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878120.442587:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878120.442589:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878120.442593:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290031744:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878120.442597:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878120.442600:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878120.442601:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878120.442604:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878120.442608:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.442616:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878120.442618:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:2.0:1788878120.442622:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878120.442639:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878120.442645:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878120.442648:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878120.442650:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.442652:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878120.442655:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.442659:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303731722 00000100:00000001:2.0:1788878120.442662:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303732265 00000100:00000001:2.0:1788878120.442665:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303731722 00000100:00000001:2.0:1788878120.442668:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303732265 00000100:00000001:2.0:1788878120.442670:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878120.442673:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.442675:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878120.442681:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878120/real 1788878120] req@ffff94e508979c00 x1875765290031744/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878120.442691:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878120.442693:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878120.442697:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878120.442700:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878120.442701:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.442704:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878120.442705:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878120.442710:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290031744/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878120.442730:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.442732:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878120.442735:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.442737:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878120.442739:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.442741:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878120.442755:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878120.442759:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878120.442764:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878120.442766:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878120.442770:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290031744:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878120.442774:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.442777:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878120.442790:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878120.442792:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878120.442795:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878120.442797:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878120.442799:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878120.442801:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878120.442803:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878120.442805:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878120.442807:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.442810:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878120.442811:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878120.442813:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878120.442815:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878120.442816:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878120.442817:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878120.442819:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878120.442821:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878120.442823:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878120.442824:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.442827:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.442831:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878120.442833:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.442835:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878120.442837:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878120.442839:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878120.442841:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878120.442842:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878120.442845:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878120.442847:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878120.442866:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878120.442869:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290031872:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878120.442873:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878120.442875:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878120.442877:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878120.442879:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878120.442882:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.442888:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878120.442890:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:2.0:1788878120.442894:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878120.442907:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878120.442912:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878120.442915:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878120.442917:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.442918:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878120.442921:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.442925:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303731722 00000100:00000001:2.0:1788878120.442929:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303732265 00000100:00000001:2.0:1788878120.442934:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303731722 00000100:00000001:2.0:1788878120.442937:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303732265 00000100:00000001:2.0:1788878120.442989:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878120.442993:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.442997:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878120.443005:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878120/real 1788878120] req@ffff94e508978a80 x1875765290031872/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878120.443014:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878120.443016:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878120.443020:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878120.443022:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878120.443023:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.443025:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878120.443027:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878120.443031:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290031872/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878120.443039:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.443041:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878120.443043:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.443045:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878120.443046:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.443047:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878120.443055:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878120.443057:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878120.443060:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878120.443061:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878120.443064:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290031872:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878120.443066:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.443068:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878120.443069:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878120.443070:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878120.443072:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878120.443073:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878120.443074:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878120.443075:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878120.443076:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878120.443078:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878120.443079:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.443080:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878120.443083:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878120.443084:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878120.443085:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878120.443086:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878125.561019:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878125.561028:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878125.561030:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878125.561034:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878125.561038:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878125.561044:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9064 00000100:00080000:2.0:1788878125.561054:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878125.561057:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878125.561059:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878125.561065:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878125.561068:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878125.561088:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878125.561090:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878125.561093:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878125.561094:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878125.561099:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878125.561102:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878125.561106:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878125.561109:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878125.561112:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561115:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878125.561118:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878125.561129:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878125.561131:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561135:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878125.561138:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878125.561159:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561172:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878125.561183:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290032000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878125.561196:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878125.561201:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561233:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878125.561237:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878125.561239:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878125.561241:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878125.561242:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878125.561244:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878125.561247:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9064 00000100:00080000:2.0:1788878125.561251:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878125.561252:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878125.561254:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878125.561257:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878125.561259:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878125.561261:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878125.561263:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878125.561265:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878125.561283:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878125.561285:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878125.561287:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878125.561290:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878125.561299:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878125.561303:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561306:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878125.561310:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878125.561313:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561318:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878125.561325:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878125.561327:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561331:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878125.561332:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878125.561336:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561342:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878125.561350:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290032128/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878125.561362:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878125.561365:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561370:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878125.561377:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878125.561381:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878125.561382:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878125.561383:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878125.561386:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878125.561390:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9064 00000100:00080000:2.0:1788878125.561393:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878125.561394:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878125.561395:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878125.561398:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878125.561398:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878125.561400:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878125.561401:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878125.561403:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878125.561405:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878125.561427:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878125.561432:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878125.561435:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878125.561440:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878125.561442:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561444:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878125.561446:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878125.561448:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561449:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878125.561454:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878125.561455:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561456:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878125.561457:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878125.561458:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561462:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878125.561466:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290032256/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878125.561472:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878125.561473:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561492:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878125.561497:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878125.561503:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878125.561504:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878125.561505:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878125.561507:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878125.561510:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9064 00000100:00080000:2.0:1788878125.561514:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878125.561519:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878125.561520:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878125.561523:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878125.561527:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878125.561529:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878125.561534:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878125.561538:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878125.561539:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878125.561560:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878125.561566:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878125.561569:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878125.561573:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878125.561576:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561577:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878125.561578:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878125.561580:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561581:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878125.561585:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878125.561589:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561590:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878125.561592:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878125.561594:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561603:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878125.561608:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290032384/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878125.561617:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878125.561620:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878125.561633:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878130.681004:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878130.681012:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878130.681014:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878130.681017:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878130.681021:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878130.681027:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9069 00000100:00080000:2.0:1788878130.681037:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878130.681039:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878130.681041:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878130.681048:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878130.681050:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878130.681052:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878130.681054:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878130.681055:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878130.681056:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878130.681059:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878130.681061:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878130.681063:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878130.681066:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878130.681068:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681070:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878130.681071:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878130.681080:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878130.681081:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681084:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878130.681087:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878130.681090:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681100:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878130.681107:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290032512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878130.681115:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878130.681118:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681131:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878130.681134:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878130.681136:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878130.681137:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878130.681138:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878130.681140:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878130.681142:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9069 00000100:00080000:2.0:1788878130.681145:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878130.681146:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878130.681148:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878130.681150:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878130.681151:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878130.681152:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878130.681154:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878130.681156:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878130.681157:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878130.681166:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878130.681167:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878130.681169:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878130.681173:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878130.681177:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681179:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878130.681185:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878130.681189:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681191:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878130.681212:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878130.681216:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681219:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878130.681221:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878130.681225:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681232:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878130.681239:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290032640/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878130.681250:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878130.681253:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681257:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878130.681259:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878130.681262:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878130.681265:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878130.681266:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878130.681269:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878130.681271:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9069 00000100:00080000:2.0:1788878130.681274:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878130.681275:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878130.681276:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878130.681279:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878130.681281:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878130.681285:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878130.681288:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878130.681291:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878130.681292:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878130.681294:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878130.681296:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878130.681297:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878130.681299:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878130.681301:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681302:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878130.681304:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878130.681305:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681307:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878130.681311:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878130.681312:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681313:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878130.681314:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878130.681315:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681320:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878130.681327:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290032768/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878130.681333:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878130.681335:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681339:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878130.681343:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878130.681349:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878130.681351:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878130.681353:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878130.681355:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878130.681359:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9069 00000100:00080000:2.0:1788878130.681365:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878130.681366:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878130.681367:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878130.681369:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878130.681370:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878130.681372:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878130.681376:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878130.681377:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878130.681378:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878130.681381:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878130.681382:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878130.681383:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878130.681385:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878130.681386:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681389:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878130.681391:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878130.681393:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681394:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878130.681400:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878130.681402:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681406:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878130.681408:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878130.681411:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681416:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878130.681423:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290032896/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878130.681440:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878130.681444:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878130.681447:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878135.801054:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878135.801062:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878135.801064:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878135.801068:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878135.801071:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878135.801077:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9074 00000100:00080000:2.0:1788878135.801085:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878135.801114:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878135.801116:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878135.801122:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878135.801124:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878135.801127:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878135.801129:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878135.801131:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878135.801133:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878135.801137:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878135.801140:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878135.801144:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878135.801148:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878135.801151:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801191:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878135.801194:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878135.801206:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878135.801208:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801212:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878135.801215:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878135.801218:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801241:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878135.801249:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290033024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878135.801257:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878135.801261:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801275:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878135.801278:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878135.801280:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878135.801281:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878135.801283:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878135.801285:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878135.801287:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9074 00000100:00080000:2.0:1788878135.801290:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878135.801291:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878135.801292:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878135.801295:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878135.801300:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878135.801303:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878135.801304:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878135.801306:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878135.801307:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878135.801309:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878135.801311:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878135.801313:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878135.801317:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878135.801319:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801323:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878135.801327:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878135.801332:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801336:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878135.801343:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878135.801346:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801351:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878135.801354:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878135.801357:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801362:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878135.801370:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290033152/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878135.801382:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878135.801387:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801390:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878135.801392:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878135.801395:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878135.801396:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878135.801399:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878135.801403:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878135.801408:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9074 00000100:00080000:2.0:1788878135.801414:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878135.801415:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878135.801416:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878135.801419:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878135.801420:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878135.801421:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878135.801423:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878135.801425:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878135.801425:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878135.801428:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878135.801429:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878135.801431:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878135.801433:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878135.801435:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801436:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878135.801438:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878135.801440:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801444:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878135.801449:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878135.801451:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801453:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878135.801455:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878135.801457:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801463:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878135.801470:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290033280/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878135.801480:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878135.801484:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801486:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878135.801488:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878135.801491:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878135.801495:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878135.801496:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878135.801498:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878135.801499:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9074 00000100:00080000:2.0:1788878135.801502:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878135.801503:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878135.801504:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878135.801506:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878135.801509:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878135.801520:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878135.801523:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878135.801528:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878135.801531:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878135.801536:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878135.801540:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878135.801543:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878135.801547:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878135.801552:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801557:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878135.801560:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878135.801565:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801568:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878135.801572:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878135.801575:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801579:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878135.801583:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878135.801586:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801592:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878135.801601:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290033408/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878135.801645:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878135.801650:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878135.801657:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878140.921020:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878140.921028:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878140.921031:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878140.921034:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878140.921037:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878140.921043:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9079 00000100:00080000:2.0:1788878140.921052:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878140.921055:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878140.921056:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878140.921064:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878140.921066:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878140.921069:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878140.921071:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878140.921072:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878140.921073:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878140.921077:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878140.921079:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878140.921081:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878140.921084:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878140.921087:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921088:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878140.921090:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878140.921100:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878140.921101:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921103:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878140.921105:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878140.921109:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921121:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878140.921131:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4000 x1875765290033536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878140.921143:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878140.921148:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921177:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878140.921182:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878140.921186:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878140.921187:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878140.921189:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878140.921192:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878140.921195:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9079 00000100:00080000:2.0:1788878140.921200:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878140.921202:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878140.921204:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878140.921208:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878140.921210:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878140.921213:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878140.921216:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878140.921218:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878140.921221:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878140.921224:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878140.921228:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878140.921230:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878140.921235:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878140.921237:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921238:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878140.921242:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878140.921244:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921246:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878140.921249:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878140.921250:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921253:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878140.921260:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878140.921263:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921272:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878140.921285:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290033664/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878140.921303:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878140.921311:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921319:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878140.921328:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878140.921335:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878140.921336:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878140.921339:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878140.921349:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878140.921353:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9079 00000100:00080000:2.0:1788878140.921355:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878140.921360:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878140.921364:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878140.921369:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878140.921371:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878140.921375:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878140.921377:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878140.921384:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878140.921388:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878140.921396:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878140.921399:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878140.921401:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878140.921405:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878140.921407:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921425:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878140.921427:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878140.921429:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921430:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878140.921435:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878140.921436:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921437:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878140.921439:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878140.921440:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921446:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878140.921453:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290033792/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878140.921471:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878140.921474:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921476:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878140.921480:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878140.921485:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878140.921487:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878140.921497:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878140.921524:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878140.921532:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9079 00000100:00080000:2.0:1788878140.921544:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878140.921548:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878140.921549:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878140.921552:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878140.921554:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878140.921557:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878140.921561:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878140.921562:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878140.921566:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878140.921569:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878140.921571:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878140.921572:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878140.921574:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878140.921575:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921576:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878140.921578:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878140.921581:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921589:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878140.921593:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878140.921597:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921602:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878140.921607:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878140.921612:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921620:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878140.921628:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290033920/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878140.921637:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878140.921640:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878140.921644:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.041288:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878146.041294:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878146.041297:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878146.041301:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878146.041306:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878146.041312:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878146.041317:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878146.041321:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878146.041328:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290034048:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878146.041337:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878146.041342:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878146.041345:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878146.041348:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878146.041353:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.041370:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878146.041373:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286523264 : -117763423028352 : ffff94e51100bb80) 00000100:00000001:2.0:1788878146.041380:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878146.041415:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878146.041427:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878146.041432:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878146.041435:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.041439:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878146.041444:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.041450:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303757865 00000100:00000001:2.0:1788878146.041464:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878146.041475:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303757865 00000100:00000001:2.0:1788878146.041484:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878146.041492:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878146.041499:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.041507:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878146.041516:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878146/real 1788878146] req@ffff94e51100bb80 x1875765290034048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878146.041556:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878146.041561:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878146.041567:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878146.041573:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878146.041577:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.041580:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878146.041584:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878146.041595:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100bb80 x1875765290034048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878146.041612:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.041618:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878146.041626:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.041631:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878146.041635:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.041641:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878146.041645:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878146.041662:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878146.041668:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878146.041671:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878146.041674:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290034048:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878146.041679:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.041681:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878146.041683:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878146.041685:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878146.041688:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878146.041690:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878146.041692:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878146.041695:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878146.041696:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878146.041699:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878146.041700:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.041702:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.041708:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878146.041709:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.041711:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878146.041712:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878146.041714:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878146.041715:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878146.041726:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878146.041728:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878146.041729:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878146.041731:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878146.041734:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290034176:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878146.041737:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878146.041739:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878146.041740:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878146.041742:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878146.041745:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.041750:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878146.041751:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:2.0:1788878146.041756:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878146.041768:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878146.041772:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878146.041774:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878146.041775:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.041777:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878146.041779:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.041783:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303757323 00000100:00000001:2.0:1788878146.041786:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303757865 00000100:00000001:2.0:1788878146.041788:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303757323 00000100:00000001:2.0:1788878146.041790:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303757865 00000100:00000001:2.0:1788878146.041791:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878146.041793:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.041794:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878146.041798:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878146/real 1788878146] req@ffff94e51100b800 x1875765290034176/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878146.041804:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878146.041805:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878146.041807:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878146.041808:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878146.041810:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.041811:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878146.041812:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878146.041815:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290034176/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878146.041826:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.041828:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878146.041830:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.041831:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878146.041832:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.041833:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878146.041845:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878146.041847:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878146.041849:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878146.041851:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878146.041853:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290034176:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878146.041857:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878146.041858:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878146.041859:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878146.041861:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878146.041862:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878146.041864:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878146.041866:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290034304:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878146.041869:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878146.041870:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878146.041871:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878146.041873:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878146.041877:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.041883:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878146.041885:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:2.0:1788878146.041889:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878146.041903:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878146.041909:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878146.041912:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878146.041913:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.041916:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878146.041944:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.041949:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303757322 00000100:00000001:2.0:1788878146.041951:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303757865 00000100:00000001:2.0:1788878146.041953:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303757322 00000100:00000001:2.0:1788878146.041955:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303757865 00000100:00000001:2.0:1788878146.041957:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878146.041958:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.041960:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878146.041964:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878146/real 1788878146] req@ffff94e51100b480 x1875765290034304/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878146.041971:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878146.041972:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878146.041975:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878146.041986:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878146.041987:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.041988:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878146.041989:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878146.041992:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290034304/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878146.042014:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.042016:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878146.042018:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.042019:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878146.042020:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.042022:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878146.042032:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878146.042035:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878146.042037:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878146.042039:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878146.042043:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290034304:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878146.042046:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878146.042047:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878146.042048:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878146.042050:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878146.042052:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878146.042053:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878146.042056:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290034432:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878146.042058:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878146.042060:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878146.042061:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878146.042063:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878146.042066:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.042070:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878146.042071:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:2.0:1788878146.042075:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878146.042109:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878146.042112:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878146.042114:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878146.042115:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.042116:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878146.042118:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.042120:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303757322 00000100:00000001:2.0:1788878146.042122:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303757865 00000100:00000001:2.0:1788878146.042123:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303757322 00000100:00000001:2.0:1788878146.042124:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303757865 00000100:00000001:2.0:1788878146.042141:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878146.042142:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.042144:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878146.042147:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878146/real 1788878146] req@ffff94e51100aa00 x1875765290034432/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878146.042157:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878146.042158:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878146.042160:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878146.042161:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878146.042162:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.042163:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878146.042164:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878146.042166:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290034432/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878146.042181:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.042182:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878146.042184:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.042185:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878146.042186:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.042187:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878146.042196:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878146.042197:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878146.042199:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878146.042200:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878146.042202:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290034432:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878146.042204:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.042206:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878146.042207:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878146.042208:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878146.042210:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878146.042211:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878146.042212:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878146.042213:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878146.042214:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878146.042215:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878146.042216:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.042218:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878146.042218:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878146.042219:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878146.042220:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878146.042221:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878146.042222:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878146.042223:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878146.042224:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878146.042225:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878146.042225:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.042226:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878146.042227:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878146.042228:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878146.042228:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878146.042229:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878146.042230:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878146.042231:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878146.042231:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878146.042232:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878146.042233:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.042234:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878146.042236:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878146.042237:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878146.042238:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878146.042239:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878151.161036:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878151.161045:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878151.161047:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878151.161050:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878151.161054:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878151.161061:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9090 00000100:00080000:2.0:1788878151.161070:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878151.161072:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878151.161073:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878151.161080:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878151.161082:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878151.161085:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878151.161087:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878151.161089:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878151.161090:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878151.161093:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878151.161095:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878151.161098:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878151.161101:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878151.161115:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161117:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878151.161120:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878151.161131:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878151.161133:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161136:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878151.161138:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878151.161144:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161183:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878151.161195:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290034560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878151.161221:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878151.161226:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161241:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878151.161244:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878151.161248:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878151.161249:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878151.161251:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878151.161254:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878151.161257:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9090 00000100:00080000:2.0:1788878151.161261:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878151.161264:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878151.161267:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878151.161272:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878151.161274:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878151.161277:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878151.161281:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878151.161289:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878151.161292:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878151.161302:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878151.161310:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878151.161315:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878151.161325:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878151.161332:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161337:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878151.161346:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878151.161354:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161359:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878151.161368:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878151.161370:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161385:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878151.161390:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878151.161393:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161403:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878151.161412:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290034688/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878151.161425:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878151.161430:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161439:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878151.161443:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878151.161447:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878151.161451:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878151.161454:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878151.161457:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878151.161460:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9090 00000100:00080000:2.0:1788878151.161482:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878151.161486:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878151.161490:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878151.161494:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878151.161497:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878151.161501:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878151.161508:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878151.161514:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878151.161517:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878151.161524:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878151.161530:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878151.161536:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878151.161542:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878151.161548:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161554:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878151.161559:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878151.161573:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161576:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878151.161581:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878151.161584:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161587:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878151.161589:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878151.161593:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161599:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878151.161604:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290034816/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878151.161615:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878151.161618:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161625:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878151.161630:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878151.161638:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878151.161642:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878151.161645:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878151.161652:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878151.161658:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9090 00000100:00080000:2.0:1788878151.161663:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878151.161668:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878151.161671:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878151.161676:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878151.161679:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878151.161683:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878151.161688:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878151.161692:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878151.161695:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878151.161703:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878151.161707:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878151.161710:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878151.161714:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878151.161719:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161722:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878151.161727:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878151.161731:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161736:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878151.161740:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878151.161742:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161744:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878151.161745:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878151.161747:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161751:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878151.161755:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290034944/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878151.161765:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878151.161771:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878151.161777:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878156.281024:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878156.281035:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878156.281040:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878156.281044:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878156.281050:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878156.281073:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9095 00000100:00080000:2.0:1788878156.281093:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878156.281097:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878156.281100:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878156.281110:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878156.281114:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878156.281118:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878156.281122:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878156.281125:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878156.281127:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878156.281133:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878156.281136:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878156.281139:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878156.281142:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878156.281144:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281147:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878156.281149:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878156.281157:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878156.281158:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281161:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878156.281164:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878156.281167:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281176:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878156.281184:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290035072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878156.281192:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878156.281197:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281210:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878156.281213:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878156.281218:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878156.281219:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878156.281220:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878156.281223:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878156.281225:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9095 00000100:00080000:2.0:1788878156.281229:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878156.281231:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878156.281233:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878156.281236:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878156.281237:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878156.281239:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878156.281242:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878156.281243:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878156.281244:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878156.281248:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878156.281253:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878156.281256:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878156.281263:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878156.281266:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281271:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878156.281277:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878156.281284:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281288:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878156.281296:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878156.281299:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281301:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878156.281304:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878156.281309:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281316:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878156.281326:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290035200/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878156.281345:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878156.281350:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281354:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878156.281357:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878156.281360:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878156.281362:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878156.281364:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878156.281370:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878156.281375:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9095 00000100:00080000:2.0:1788878156.281381:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878156.281384:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878156.281386:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878156.281389:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878156.281391:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878156.281393:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878156.281395:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878156.281397:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878156.281398:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878156.281402:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878156.281404:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878156.281406:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878156.281409:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878156.281411:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281414:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878156.281418:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878156.281423:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281424:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878156.281430:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878156.281432:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281437:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878156.281439:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878156.281441:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281450:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878156.281459:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290035328/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878156.281469:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878156.281472:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281474:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878156.281477:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878156.281482:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878156.281483:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878156.281484:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878156.281487:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878156.281492:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9095 00000100:00080000:2.0:1788878156.281500:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878156.281502:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878156.281504:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878156.281511:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878156.281514:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878156.281519:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878156.281523:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878156.281528:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878156.281531:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878156.281537:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878156.281541:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878156.281544:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878156.281548:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878156.281554:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281559:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878156.281562:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878156.281568:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281573:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878156.281580:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878156.281585:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281588:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878156.281592:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878156.281596:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281605:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878156.281614:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290035456/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878156.281631:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878156.281635:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878156.281640:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878161.400977:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878161.400985:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878161.400987:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878161.400989:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878161.400993:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878161.400998:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9100 00000100:00080000:2.0:1788878161.401007:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878161.401009:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878161.401010:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878161.401017:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878161.401019:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878161.401023:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878161.401024:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878161.401026:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878161.401027:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878161.401030:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878161.401032:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878161.401034:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878161.401038:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878161.401040:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401041:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878161.401043:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878161.401053:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878161.401055:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401073:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878161.401075:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878161.401078:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401089:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878161.401097:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290035584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878161.401104:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878161.401108:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401121:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878161.401124:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878161.401127:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878161.401128:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878161.401129:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878161.401131:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878161.401133:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9100 00000100:00080000:2.0:1788878161.401136:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878161.401137:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878161.401139:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878161.401141:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878161.401143:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878161.401145:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878161.401147:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878161.401148:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878161.401150:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878161.401152:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878161.401154:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878161.401156:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878161.401163:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878161.401168:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401171:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878161.401178:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878161.401181:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401186:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878161.401192:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878161.401195:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401197:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878161.401199:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878161.401203:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401210:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878161.401219:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290035712/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878161.401232:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878161.401235:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401240:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878161.401246:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878161.401252:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878161.401254:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878161.401256:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878161.401259:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878161.401262:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9100 00000100:00080000:2.0:1788878161.401267:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878161.401270:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878161.401271:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878161.401275:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878161.401278:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878161.401283:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878161.401286:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878161.401289:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878161.401294:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878161.401302:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878161.401310:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878161.401317:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878161.401320:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878161.401323:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401327:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878161.401329:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878161.401331:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401333:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878161.401339:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878161.401343:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401348:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878161.401352:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878161.401356:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401367:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878161.401379:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290035840/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878161.401399:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878161.401403:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401408:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878161.401413:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878161.401423:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878161.401426:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878161.401430:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878161.401434:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878161.401438:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9100 00000100:00080000:2.0:1788878161.401444:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878161.401451:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878161.401455:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878161.401463:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878161.401468:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878161.401474:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878161.401481:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878161.401487:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878161.401492:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878161.401501:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878161.401507:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878161.401510:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878161.401514:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878161.401519:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401523:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878161.401526:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878161.401528:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401529:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878161.401533:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878161.401536:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401539:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878161.401542:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878161.401546:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401555:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878161.401565:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290035968/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878161.401580:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878161.401583:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878161.401589:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878176.761105:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878176.761114:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878176.761116:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878176.761119:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878176.761123:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878176.761130:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9115 00000100:00080000:2.0:1788878176.761140:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878176.761143:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878176.761145:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878176.761152:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878176.761154:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878176.761157:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878176.761159:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878176.761161:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878176.761162:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878176.761166:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878176.761168:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878176.761170:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878176.761173:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878176.761176:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761178:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878176.761180:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878176.761190:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878176.761192:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761195:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878176.761197:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878176.761201:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761212:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878176.761220:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4380 x1875765290037120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878176.761230:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878176.761234:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761253:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878176.761256:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878176.761264:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878176.761265:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878176.761267:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878176.761269:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878176.761271:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9115 00000100:00080000:2.0:1788878176.761275:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878176.761277:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878176.761278:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878176.761281:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878176.761282:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878176.761284:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878176.761286:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878176.761288:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878176.761290:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878176.761293:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878176.761295:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878176.761297:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878176.761302:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878176.761304:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761305:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878176.761309:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878176.761313:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761317:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878176.761323:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878176.761327:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761331:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878176.761334:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878176.761336:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761343:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878176.761351:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5f80 x1875765290037248/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878176.761370:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878176.761373:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761379:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878176.761383:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878176.761388:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878176.761393:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878176.761395:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878176.761398:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878176.761401:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9115 00000100:00080000:2.0:1788878176.761404:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878176.761405:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878176.761406:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878176.761408:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878176.761409:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878176.761411:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878176.761413:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878176.761415:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878176.761420:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878176.761429:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878176.761434:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878176.761435:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878176.761438:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878176.761440:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761441:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878176.761443:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878176.761445:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761447:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878176.761452:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878176.761453:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761454:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878176.761456:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878176.761457:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761462:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878176.761466:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290037376/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878176.761480:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878176.761482:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761486:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878176.761488:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878176.761492:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878176.761494:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878176.761500:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878176.761507:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878176.761514:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9115 00000100:00080000:2.0:1788878176.761522:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878176.761528:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878176.761530:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878176.761533:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878176.761533:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878176.761535:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878176.761537:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878176.761542:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878176.761544:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878176.761547:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878176.761549:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878176.761550:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878176.761552:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878176.761555:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761561:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878176.761563:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878176.761565:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761570:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878176.761575:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878176.761579:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761581:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878176.761584:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878176.761588:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761592:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878176.761598:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290037504/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878176.761608:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878176.761611:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878176.761614:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878181.881128:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878181.881140:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878181.881143:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878181.881146:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878181.881151:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878181.881157:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9120 00000100:00080000:2.0:1788878181.881167:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878181.881169:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878181.881171:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878181.881178:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878181.881181:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878181.881184:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878181.881186:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878181.881188:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878181.881189:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878181.881193:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878181.881195:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878181.881197:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878181.881201:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878181.881203:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881206:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878181.881208:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878181.881219:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878181.881220:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881223:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878181.881226:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878181.881230:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881241:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878181.881250:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290037632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878181.881260:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878181.881264:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881280:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878181.881283:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878181.881286:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878181.881287:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878181.881289:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878181.881307:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878181.881309:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9120 00000100:00080000:2.0:1788878181.881312:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878181.881314:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878181.881315:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878181.881317:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878181.881318:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878181.881320:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878181.881322:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878181.881324:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878181.881325:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878181.881328:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878181.881330:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878181.881331:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878181.881337:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878181.881339:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881341:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878181.881345:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878181.881348:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881351:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878181.881354:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878181.881356:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881359:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878181.881363:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878181.881365:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881373:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878181.881382:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290037760/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878181.881392:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878181.881397:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881403:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878181.881411:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878181.881418:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878181.881421:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878181.881424:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878181.881432:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878181.881438:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9120 00000100:00080000:2.0:1788878181.881442:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878181.881447:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878181.881451:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878181.881454:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878181.881457:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878181.881459:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878181.881465:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878181.881469:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878181.881470:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878181.881473:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878181.881476:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878181.881477:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878181.881480:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878181.881482:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881487:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878181.881491:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878181.881495:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881499:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878181.881503:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878181.881504:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881506:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878181.881507:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878181.881508:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881513:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878181.881516:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290037888/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878181.881523:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878181.881525:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881527:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878181.881529:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878181.881537:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878181.881542:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878181.881547:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878181.881552:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878181.881558:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9120 00000100:00080000:2.0:1788878181.881567:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878181.881575:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878181.881579:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878181.881588:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878181.881592:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878181.881596:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878181.881602:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878181.881605:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878181.881607:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878181.881611:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878181.881615:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878181.881630:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878181.881635:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878181.881642:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881646:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878181.881650:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878181.881658:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881665:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878181.881675:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878181.881679:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881683:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878181.881685:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878181.881691:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881697:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878181.881709:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290038016/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878181.881735:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878181.881742:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878181.881751:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878187.001010:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878187.001021:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878187.001024:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878187.001028:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878187.001033:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878187.001040:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9125 00000100:00080000:2.0:1788878187.001050:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878187.001052:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878187.001053:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878187.001059:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878187.001061:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878187.001064:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878187.001065:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878187.001067:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878187.001068:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878187.001072:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878187.001074:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878187.001076:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878187.001079:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878187.001081:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001083:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878187.001085:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878187.001094:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878187.001095:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001098:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878187.001100:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878187.001104:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001113:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878187.001120:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290038144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878187.001128:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878187.001132:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001144:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878187.001146:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878187.001149:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878187.001150:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878187.001151:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878187.001153:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878187.001155:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9125 00000100:00080000:2.0:1788878187.001157:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878187.001158:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878187.001159:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878187.001161:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878187.001162:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878187.001164:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878187.001165:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878187.001167:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878187.001169:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878187.001171:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878187.001173:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878187.001175:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878187.001179:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878187.001181:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001182:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878187.001186:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878187.001190:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001193:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878187.001197:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878187.001198:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001202:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878187.001204:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878187.001205:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001211:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878187.001219:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290038272/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878187.001228:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878187.001231:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001235:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878187.001241:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878187.001244:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878187.001246:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878187.001248:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878187.001251:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878187.001255:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9125 00000100:00080000:2.0:1788878187.001259:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878187.001260:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878187.001263:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878187.001266:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878187.001268:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878187.001271:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878187.001274:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878187.001279:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878187.001282:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878187.001287:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878187.001288:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878187.001289:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878187.001291:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878187.001295:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001298:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878187.001299:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878187.001300:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001301:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878187.001305:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878187.001305:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001307:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878187.001308:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878187.001309:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001313:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878187.001319:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290038400/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878187.001328:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878187.001329:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001331:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878187.001333:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878187.001335:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878187.001336:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878187.001337:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878187.001339:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878187.001340:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9125 00000100:00080000:2.0:1788878187.001343:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878187.001345:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878187.001346:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878187.001348:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878187.001349:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878187.001350:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878187.001352:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878187.001353:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878187.001354:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878187.001360:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878187.001364:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878187.001365:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878187.001367:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878187.001369:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001370:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878187.001372:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878187.001373:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001377:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878187.001381:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878187.001383:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001387:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878187.001389:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878187.001392:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001397:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878187.001403:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290038528/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878187.001411:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878187.001413:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878187.001414:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878192.120988:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878192.120996:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878192.120998:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878192.121001:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878192.121006:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878192.121012:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9130 00000100:00080000:2.0:1788878192.121023:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878192.121026:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878192.121028:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878192.121037:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878192.121040:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878192.121043:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878192.121046:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878192.121049:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878192.121053:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878192.121062:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878192.121065:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878192.121069:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878192.121073:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878192.121076:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121078:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878192.121081:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878192.121092:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878192.121093:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121096:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878192.121099:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878192.121102:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121112:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878192.121122:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290038656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878192.121132:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878192.121137:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121154:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878192.121157:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878192.121160:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878192.121161:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878192.121163:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878192.121166:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878192.121169:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9130 00000100:00080000:2.0:1788878192.121173:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878192.121175:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878192.121177:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878192.121182:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878192.121184:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878192.121188:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878192.121192:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878192.121195:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878192.121196:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878192.121202:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878192.121208:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878192.121211:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878192.121218:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878192.121224:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121229:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878192.121239:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878192.121252:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121261:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878192.121274:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878192.121281:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121286:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878192.121287:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878192.121296:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121310:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878192.121322:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290038784/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878192.121333:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878192.121338:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121347:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878192.121355:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878192.121359:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878192.121360:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878192.121362:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878192.121365:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878192.121367:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9130 00000100:00080000:2.0:1788878192.121374:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878192.121376:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878192.121379:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878192.121383:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878192.121384:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878192.121387:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878192.121389:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878192.121391:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878192.121392:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878192.121396:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878192.121400:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878192.121406:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878192.121413:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878192.121418:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121421:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878192.121425:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878192.121435:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121447:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878192.121458:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878192.121464:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121470:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878192.121472:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878192.121476:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121487:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878192.121493:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290038912/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878192.121521:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878192.121525:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121534:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878192.121537:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878192.121543:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878192.121546:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878192.121553:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878192.121569:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878192.121579:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9130 00000100:00080000:2.0:1788878192.121591:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878192.121600:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878192.121608:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878192.121618:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878192.121628:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878192.121634:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878192.121642:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878192.121654:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878192.121660:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878192.121678:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878192.121688:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878192.121694:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878192.121707:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878192.121715:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121742:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878192.121753:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878192.121766:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121774:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878192.121788:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878192.121795:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121803:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878192.121813:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878192.121817:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121824:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878192.121838:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290039040/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878192.121867:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878192.121873:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878192.121881:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878214.393013:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788878214.393016:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788878214.393018:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788878217.721247:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878217.721255:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878217.721259:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878217.721263:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878217.721274:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878217.721290:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878217.721300:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878217.721312:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878217.721335:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290041216:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878217.721349:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878217.721355:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878217.721358:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878217.721363:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878217.721374:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878217.721391:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878217.721394:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286512512 : -117763423039104 : ffff94e511009180) 00000100:00000001:2.0:1788878217.721401:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878217.721452:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878217.721468:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303829545 00000100:00000001:2.0:1788878217.721480:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878217.721503:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303829545 00000100:00000001:2.0:1788878217.721517:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878217.721527:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878217.721547:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878217.721561:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878217.721572:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878217.721588:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878217.721594:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788878217.721606:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878217.721617:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878217.721626:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878217.721634:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878217.721646:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878217.721663:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878217.721666:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878217.721668:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878217.721674:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290041344:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878217.721680:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878217.721686:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878217.721690:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878217.721694:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878217.721697:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878217.721716:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878217.721721:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286517888 : -117763423033728 : ffff94e51100a680) 00000100:00000001:2.0:1788878217.721731:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878217.721747:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878217.721752:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303829003 00000100:00000001:2.0:1788878217.721757:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303829545 00000100:00000001:2.0:1788878217.721763:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303829003 00000100:00000001:2.0:1788878217.721776:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303829545 00000100:00000001:2.0:1788878217.721789:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878217.721819:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878217.721825:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878217.721837:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878217.721843:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878217.721848:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878217.721852:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878217.721864:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878217.721870:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290041472:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878217.721878:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878217.721881:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878217.721885:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878217.721886:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878217.721889:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878217.721895:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878217.721898:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286516096 : -117763423035520 : ffff94e511009f80) 00000100:00000001:2.0:1788878217.721942:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878217.721962:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878217.721967:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303829002 00000100:00000001:2.0:1788878217.721971:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303829545 00000100:00000001:2.0:1788878217.721974:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303829002 00000100:00000001:2.0:1788878217.721976:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303829545 00000100:00000001:2.0:1788878217.721978:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878217.721980:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878217.721983:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878217.721985:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878217.721988:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878217.721992:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788878217.722004:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878217.722008:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878217.722011:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878217.722014:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878217.722017:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878217.722022:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878217.722026:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878217.722029:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878217.722035:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290041600:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878217.722039:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878217.722043:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878217.722047:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878217.722049:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878217.722054:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878217.722061:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878217.722063:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508928 : -117763423042688 : ffff94e511008380) 00000100:00000001:2.0:1788878217.722068:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878217.722082:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878217.722086:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303829002 00000100:00000001:2.0:1788878217.722088:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303829545 00000100:00000001:2.0:1788878217.722090:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303829002 00000100:00000001:2.0:1788878217.722092:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303829545 00000100:00000001:2.0:1788878217.722094:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878217.722096:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878217.722098:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878217.722100:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878217.722103:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878217.722104:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788878217.722106:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878217.722107:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878217.722108:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878217.722111:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878218.489155:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788878218.489161:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788878218.489165:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788878220.793236:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:2.0:1788878220.793293:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788878220.793322:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788878220.793338:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878220.793364:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878220.793367:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878220.793381:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:2.0:1788878220.793393:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788878220.793400:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878220.793404:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878220.793407:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878220.793409:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:2.0:1788878220.793418:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788878220.793436:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878220.793440:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878220.793443:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878220.793448:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:2.0:1788878220.793456:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788878220.793469:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878220.793471:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878220.793473:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878220.793474:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:2.0:1788878222.585078:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788878222.585082:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788878222.585085:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788878226.681065:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788878226.681071:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788878226.681074:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788878230.777162:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788878230.777168:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788878230.777173:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878233.080999:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878233.081010:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878233.081013:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878233.081017:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878233.081021:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878233.081028:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9171 00000100:00080000:2.0:1788878233.081040:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878233.081042:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878233.081044:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878233.081053:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878233.081055:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878233.081059:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878233.081061:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878233.081063:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878233.081064:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878233.081069:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878233.081071:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878233.081074:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878233.081077:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878233.081079:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081081:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878233.081084:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878233.081095:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878233.081096:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081099:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878233.081101:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878233.081106:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081119:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878233.081128:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290042752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878233.081137:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878233.081141:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081155:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878233.081157:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878233.081160:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878233.081161:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878233.081162:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878233.081165:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878233.081167:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9171 00000100:00080000:2.0:1788878233.081169:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878233.081170:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878233.081171:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878233.081174:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878233.081175:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878233.081177:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878233.081179:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878233.081181:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878233.081183:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878233.081185:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878233.081188:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878233.081189:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878233.081195:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878233.081199:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081201:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878233.081207:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878233.081210:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081213:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878233.081219:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878233.081223:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081224:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878233.081228:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878233.081231:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081238:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878233.081246:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4000 x1875765290042880/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878233.081258:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878233.081261:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081265:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878233.081267:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878233.081273:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878233.081273:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878233.081274:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878233.081276:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878233.081279:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9171 00000100:00080000:2.0:1788878233.081281:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878233.081282:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878233.081283:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878233.081286:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878233.081287:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878233.081290:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878233.081294:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878233.081297:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878233.081299:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878233.081301:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878233.081302:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878233.081303:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878233.081305:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878233.081308:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081310:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878233.081311:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878233.081313:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081314:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878233.081319:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878233.081320:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081321:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878233.081323:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878233.081324:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081328:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878233.081331:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290043008/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878233.081337:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878233.081338:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081340:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878233.081342:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878233.081344:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878233.081345:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878233.081346:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878233.081348:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878233.081354:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9171 00000100:00080000:2.0:1788878233.081360:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878233.081362:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878233.081363:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878233.081365:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878233.081366:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878233.081367:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878233.081368:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878233.081370:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878233.081371:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878233.081373:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878233.081375:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878233.081376:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878233.081380:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878233.081382:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081384:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878233.081387:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878233.081393:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081397:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878233.081400:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878233.081402:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081407:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878233.081408:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878233.081410:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081416:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878233.081420:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290043136/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878233.081427:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878233.081429:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878233.081431:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878234.873097:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788878234.873101:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788878234.873104:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:2.0:1788878238.969051:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788878238.969054:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788878238.969058:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 02000000:00000001:3.0:1788878243.322150:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878243.322159:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878243.322161:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:3.0:1788878243.322166:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878243.322184:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878243.322190:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878243.322193:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878243.322194:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878243.322196:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878243.322199:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878243.322204:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303854604 00000100:00000001:3.0:1788878243.322207:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303855145 00000100:00000001:3.0:1788878243.322210:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303854604 00000100:00000001:3.0:1788878243.322212:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303855145 00000100:00000001:3.0:1788878243.322215:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878243.322217:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878243.322219:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878243.322224:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878243/real 1788878243] req@ffff94e50897bb80 x1875765290043904/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878243.322233:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878243.322234:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878243.322237:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878243.322239:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878243.322240:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322242:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878243.322244:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878243.322248:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290043904/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878243.322256:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322258:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878243.322260:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322262:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878243.322263:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322266:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878243.322279:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878243.322282:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878243.322285:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878243.322288:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878243.322291:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290043904:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878243.322295:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878243.322297:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878243.322298:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878243.322300:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878243.322302:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878243.322305:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878243.322308:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290044032:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878243.322311:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878243.322313:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878243.322315:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878243.322316:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878243.322319:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878243.322324:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878243.322325:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:3.0:1788878243.322328:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878243.322339:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878243.322343:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878243.322345:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878243.322346:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878243.322348:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878243.322351:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878243.322354:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303854602 00000100:00000001:3.0:1788878243.322356:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303855145 00000100:00000001:3.0:1788878243.322359:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303854602 00000100:00000001:3.0:1788878243.322361:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303855145 00000100:00000001:3.0:1788878243.322365:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878243.322367:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878243.322369:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878243.322373:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878243/real 1788878243] req@ffff94e50897ad80 x1875765290044032/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878243.322382:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878243.322383:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878243.322385:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878243.322386:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878243.322388:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322389:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878243.322391:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878243.322395:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290044032/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878243.322402:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322404:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878243.322407:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322408:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878243.322409:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322411:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878243.322423:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878243.322425:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878243.322428:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878243.322430:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878243.322433:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290044032:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878243.322436:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878243.322438:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878243.322439:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878243.322441:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878243.322443:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878243.322446:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878243.322449:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290044160:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878243.322453:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878243.322454:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878243.322456:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878243.322458:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878243.322460:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878243.322464:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878243.322466:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:3.0:1788878243.322469:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878243.322478:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878243.322481:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878243.322484:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878243.322485:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878243.322487:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878243.322489:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878243.322492:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303854602 00000100:00000001:3.0:1788878243.322495:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303855145 00000100:00000001:3.0:1788878243.322497:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303854602 00000100:00000001:3.0:1788878243.322517:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303855145 00000100:00000001:3.0:1788878243.322520:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878243.322522:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878243.322524:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878243.322528:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878243/real 1788878243] req@ffff94e508978e00 x1875765290044160/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878243.322537:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878243.322538:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878243.322541:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878243.322542:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878243.322543:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322545:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878243.322547:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878243.322551:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290044160/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878243.322566:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322569:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878243.322572:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322573:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878243.322575:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322576:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878243.322587:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878243.322590:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878243.322593:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878243.322595:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878243.322598:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290044160:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878243.322602:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322605:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878243.322606:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878243.322608:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878243.322611:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878243.322612:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878243.322614:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878243.322617:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878243.322618:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878243.322622:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878243.322623:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322625:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878243.322627:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878243.322628:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878243.322630:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878243.322631:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878243.322633:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878243.322635:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878243.322636:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878243.322638:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878243.322640:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322641:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878243.322643:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878243.322645:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878243.322646:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878243.322648:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878243.322649:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878243.322651:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878243.322652:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878243.322654:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878243.322656:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322658:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878243.322661:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878243.322663:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878243.322665:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878243.322667:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878245.113173:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878245.113178:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878245.113183:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788878248.441437:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878248.441444:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878248.441449:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878248.441455:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878248.441459:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878248.441467:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878248.441473:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878248.441477:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878248.441492:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290044288:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878248.441505:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878248.441512:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878248.441516:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878248.441522:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878248.441530:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.441576:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878248.441582:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 00000100:00000001:3.0:1788878248.441595:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878248.441726:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878248.441749:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878248.441774:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878248.441779:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.441784:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878248.441795:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.441804:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303860265 00000100:00000001:3.0:1788878248.441822:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878248.441844:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303860265 00000100:00000001:3.0:1788878248.441852:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878248.441858:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878248.441865:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.441872:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878248.441884:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878248/real 1788878248] req@ffff94e508979180 x1875765290044288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878248.441977:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878248.441985:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878248.442013:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878248.442019:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878248.442021:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442025:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878248.442028:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878248.442035:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290044288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878248.442043:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442047:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878248.442049:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442051:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878248.442052:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442055:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878248.442056:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878248.442059:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878248.442063:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878248.442065:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878248.442068:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290044288:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878248.442073:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442076:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878248.442078:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878248.442081:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878248.442084:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878248.442087:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878248.442110:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878248.442116:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878248.442118:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878248.442122:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878248.442124:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442126:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442135:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878248.442137:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.442139:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878248.442141:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878248.442143:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878248.442145:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878248.442146:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878248.442148:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878248.442151:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878248.442153:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878248.442156:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290044416:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878248.442159:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878248.442161:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878248.442164:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878248.442167:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878248.442171:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.442181:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878248.442184:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000100:00000001:3.0:1788878248.442191:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878248.442211:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878248.442218:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878248.442222:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878248.442224:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.442227:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878248.442231:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.442238:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303859724 00000100:00000001:3.0:1788878248.442243:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303860265 00000100:00000001:3.0:1788878248.442247:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303859724 00000100:00000001:3.0:1788878248.442251:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303860265 00000100:00000001:3.0:1788878248.442253:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878248.442272:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.442274:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878248.442279:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878248/real 1788878248] req@ffff94e50897b800 x1875765290044416/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878248.442287:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878248.442288:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878248.442300:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878248.442302:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878248.442303:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442305:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878248.442307:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878248.442310:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290044416/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878248.442317:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442319:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878248.442321:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442322:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878248.442323:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442325:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878248.442342:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878248.442344:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878248.442346:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878248.442348:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878248.442351:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290044416:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878248.442354:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878248.442355:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878248.442357:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878248.442359:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878248.442361:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878248.442363:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878248.442365:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290044544:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878248.442368:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878248.442370:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878248.442371:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878248.442373:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878248.442376:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.442382:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878248.442384:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 00000100:00000001:3.0:1788878248.442388:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878248.442399:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878248.442403:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878248.442405:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878248.442406:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.442408:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878248.442410:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.442413:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303859722 00000100:00000001:3.0:1788878248.442416:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303860265 00000100:00000001:3.0:1788878248.442418:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303859722 00000100:00000001:3.0:1788878248.442420:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303860265 00000100:00000001:3.0:1788878248.442422:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878248.442423:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.442425:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878248.442428:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878248/real 1788878248] req@ffff94e508978380 x1875765290044544/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878248.442434:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878248.442435:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878248.442437:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878248.442438:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878248.442439:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442441:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878248.442443:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878248.442446:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290044544/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878248.442453:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442454:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878248.442456:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442457:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878248.442458:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442459:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878248.442474:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878248.442476:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878248.442478:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878248.442480:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878248.442482:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290044544:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878248.442485:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878248.442486:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878248.442487:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878248.442489:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878248.442491:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878248.442493:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878248.442495:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290044672:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878248.442497:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878248.442499:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878248.442500:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878248.442501:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878248.442503:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.442507:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878248.442508:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:3.0:1788878248.442511:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878248.442518:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878248.442521:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878248.442523:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878248.442524:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.442526:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878248.442528:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.442547:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303859722 00000100:00000001:3.0:1788878248.442550:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303860265 00000100:00000001:3.0:1788878248.442552:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303859722 00000100:00000001:3.0:1788878248.442554:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303860265 00000100:00000001:3.0:1788878248.442557:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878248.442558:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.442560:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878248.442563:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878248/real 1788878248] req@ffff94e508978e00 x1875765290044672/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878248.442570:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878248.442571:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878248.442573:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878248.442574:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878248.442576:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442577:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878248.442579:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878248.442582:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290044672/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878248.442588:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442590:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878248.442592:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442593:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878248.442594:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442596:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878248.442612:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878248.442614:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878248.442617:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878248.442619:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878248.442623:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290044672:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878248.442626:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442628:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878248.442629:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878248.442630:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878248.442632:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878248.442634:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878248.442635:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878248.442637:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878248.442638:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878248.442642:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878248.442643:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442645:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878248.442646:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878248.442647:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878248.442649:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878248.442650:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878248.442651:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878248.442653:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878248.442654:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878248.442656:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878248.442657:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442658:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878248.442659:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878248.442660:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878248.442661:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878248.442662:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878248.442663:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878248.442686:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878248.442687:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878248.442689:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878248.442690:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442692:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878248.442695:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878248.442697:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878248.442699:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878248.442700:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878249.209275:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878249.209282:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878249.209287:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1788878253.305108:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878253.305112:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878253.305115:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878253.561089:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878253.561111:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878253.561114:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878253.561118:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878253.561122:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878253.561130:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9192 00000100:00080000:2.0:1788878253.561144:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878253.561146:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878253.561148:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878253.561158:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878253.561160:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878253.561164:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878253.561166:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878253.561168:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878253.561169:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878253.561172:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878253.561174:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878253.561176:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878253.561180:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878253.561182:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878253.561185:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878253.561188:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878253.561204:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878253.561206:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878253.561211:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878253.561214:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878253.561220:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878253.561239:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878253.561253:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290044800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878253.561262:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878253.561268:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878253.561290:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878253.561294:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878253.561298:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878253.561300:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878253.561302:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878253.561307:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878253.561311:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9192 00000100:00080000:2.0:1788878253.561317:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878253.561320:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878253.561322:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878253.561326:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878253.561328:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788878253.561329:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788878253.561331:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878253.561337:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878253.561338:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878253.561343:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878253.561344:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878253.561346:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878253.561348:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878253.561357:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878253.561360:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878253.561368:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878253.561381:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878253.561387:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878253.561389:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878253.561396:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878253.561396:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878253.561413:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788878253.561415:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788878253.561423:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878253.561430:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878253.561430:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290044800:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878253.561443:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.561451:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878253.561454:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878253.561458:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878253.561458:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788878253.561468:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878253.561470:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878253.561478:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878253.561479:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878253.561490:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878253.561497:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878253.561499:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878253.561510:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290044928/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878253.561518:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878253.561519:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878253.561523:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878253.561526:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878253.561530:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878253.561531:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788878253.561532:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788878253.561533:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878253.561536:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00000001:2.0:1788878253.561538:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788878253.561551:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788878253.561553:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9192 00000100:00080000:2.0:1788878253.561576:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878253.561580:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878253.561582:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878253.561584:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878253.561586:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878253.561587:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878253.561589:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878253.561591:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878253.561592:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878253.561595:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878253.561597:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878253.561599:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878253.561601:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878253.561602:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878253.561604:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878253.561615:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878253.561617:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878253.561618:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878253.561622:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878253.561624:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878253.561625:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878253.561626:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878253.561628:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878253.561635:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788878253.561635:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1788878253.561663:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290045056/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878253.561671:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878253.561673:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878253.561674:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878253.561681:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878253.561685:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788878253.561687:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788878253.561692:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878253.561694:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.561697:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878253.561700:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788878253.561703:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878253.561704:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878253.561715:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788878253.561715:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303865385 00000100:00080000:2.0:1788878253.561722:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9192 00000100:00080000:2.0:1788878253.561727:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878253.561729:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878253.561731:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878253.561732:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1788878253.561735:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878253.561737:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878253.561740:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878253.561742:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878253.561744:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878253.561745:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878253.561747:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878253.561748:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878253.561749:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878253.561752:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878253.561753:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878253.561754:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878253.561757:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788878253.561757:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303865385 00000008:00000001:2.0:1788878253.561760:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878253.561762:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878253.561765:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878253.561768:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878253.561770:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878253.561771:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878253.561773:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.561776:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878253.561779:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.561788:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1788878253.561789:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290045184/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878253.561804:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878253.561806:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878253.561810:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878253.561815:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.561832:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878253.561848:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878253/real 1788878253] req@ffff94e5045e7480 x1875765290044800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878253.561863:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878253.561865:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878253.561870:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878253.561873:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878253.561875:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.561878:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878253.561880:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878253.561886:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290044800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878253.561894:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.561898:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878253.561901:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.561903:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878253.561904:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.561907:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878253.561909:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878253.561913:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878253.561952:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878253.561957:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878253.561963:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290044800:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878253.561971:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.561974:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878253.561976:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878253.561980:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878253.561987:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878253.561991:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878253.561994:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878253.562000:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878253.562002:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878253.562007:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878253.562011:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562015:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562024:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878253.562027:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.562032:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878253.562035:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878253.562037:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878253.562040:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878253.562042:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878253.562046:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878253.562050:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878253.562054:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878253.562061:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290044928:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878253.562066:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878253.562069:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878253.562071:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878253.562073:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878253.562077:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.562101:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878253.562103:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:3.0:1788878253.562107:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878253.562123:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878253.562127:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878253.562129:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878253.562131:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.562132:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878253.562135:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.562138:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303864844 00000100:00000001:3.0:1788878253.562141:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303865385 00000100:00000001:3.0:1788878253.562143:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303864844 00000100:00000001:3.0:1788878253.562145:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303865385 00000100:00000001:3.0:1788878253.562147:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878253.562149:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.562150:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878253.562154:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878253/real 1788878253] req@ffff94e5045e4700 x1875765290044928/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878253.562162:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878253.562163:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878253.562183:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878253.562185:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878253.562187:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562189:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878253.562191:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878253.562195:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290044928/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878253.562202:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562205:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878253.562207:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562209:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878253.562210:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562212:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878253.562227:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878253.562230:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878253.562233:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878253.562236:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878253.562240:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290044928:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878253.562244:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878253.562245:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878253.562247:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878253.562250:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878253.562251:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878253.562254:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878253.562257:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290045056:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878253.562260:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878253.562263:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878253.562265:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878253.562267:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878253.562270:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.562276:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878253.562278:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 00000100:00000001:3.0:1788878253.562282:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878253.562294:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878253.562298:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878253.562300:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878253.562302:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.562303:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878253.562306:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.562309:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303864842 00000100:00000001:3.0:1788878253.562312:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303865385 00000100:00000001:3.0:1788878253.562314:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303864842 00000100:00000001:3.0:1788878253.562317:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303865385 00000100:00000001:3.0:1788878253.562319:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878253.562321:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.562323:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878253.562327:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878253/real 1788878253] req@ffff94e5045e5180 x1875765290045056/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878253.562335:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878253.562336:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878253.562338:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878253.562340:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878253.562341:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562343:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878253.562344:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878253.562348:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290045056/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878253.562356:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562358:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878253.562361:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562363:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878253.562364:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562366:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878253.562378:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878253.562381:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878253.562384:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878253.562387:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878253.562390:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290045056:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878253.562393:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878253.562395:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878253.562396:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878253.562399:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878253.562400:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878253.562402:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878253.562405:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290045184:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878253.562408:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878253.562410:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878253.562412:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878253.562414:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878253.562417:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.562421:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878253.562422:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:3.0:1788878253.562426:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878253.562435:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878253.562439:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878253.562441:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878253.562442:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.562444:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878253.562447:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.562450:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303864842 00000100:00000001:3.0:1788878253.562452:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303865385 00000100:00000001:3.0:1788878253.562455:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303864842 00000100:00000001:3.0:1788878253.562458:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303865385 00000100:00000001:3.0:1788878253.562460:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878253.562462:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.562464:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878253.562468:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878253/real 1788878253] req@ffff94e5045e5880 x1875765290045184/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878253.562475:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878253.562476:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878253.562478:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878253.562480:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878253.562481:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562483:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878253.562484:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878253.562488:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290045184/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878253.562495:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562497:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878253.562500:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562502:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878253.562503:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562505:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878253.562517:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878253.562519:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878253.562523:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878253.562525:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878253.562528:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290045184:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878253.562532:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562534:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878253.562536:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878253.562538:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878253.562540:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878253.562542:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878253.562544:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878253.562546:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878253.562548:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878253.562566:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878253.562568:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562569:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878253.562570:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878253.562571:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878253.562573:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878253.562574:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878253.562575:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878253.562576:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878253.562577:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878253.562579:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878253.562580:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562581:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878253.562582:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878253.562583:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878253.562584:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878253.562586:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878253.562587:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878253.562590:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878253.562592:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878253.562593:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878253.562595:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562598:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878253.562602:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878253.562604:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878253.562606:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878253.562608:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878257.401189:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878257.401197:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878257.401202:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878258.680997:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878258.681009:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878258.681011:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878258.681016:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878258.681021:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878258.681029:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9197 00000100:00080000:2.0:1788878258.681042:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878258.681045:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878258.681047:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878258.681056:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878258.681059:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878258.681063:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878258.681065:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878258.681067:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878258.681068:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878258.681072:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878258.681074:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878258.681078:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878258.681082:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878258.681085:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878258.681087:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878258.681090:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878258.681105:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878258.681106:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878258.681110:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878258.681113:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878258.681118:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878258.681133:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878258.681142:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290045312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878258.681151:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878258.681156:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878258.681173:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878258.681176:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878258.681179:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878258.681180:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878258.681181:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878258.681184:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878258.681185:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9197 00000100:00080000:2.0:1788878258.681188:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878258.681189:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878258.681191:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878258.681193:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878258.681195:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878258.681197:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878258.681199:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878258.681200:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878258.681201:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878258.681204:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878258.681207:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878258.681208:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788878258.681210:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788878258.681214:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878258.681216:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788878258.681218:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681221:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878258.681222:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878258.681226:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:2.0:1788878258.681228:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788878258.681229:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788878258.681233:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878258.681235:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878258.681236:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878258.681239:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878258.681240:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878258.681241:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878258.681246:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878258.681247:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878258.681248:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878258.681252:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878258.681257:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290045312:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878258.681258:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681265:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:2.0:1788878258.681266:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290045440/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788878258.681272:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788878258.681274:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788878258.681275:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878258.681276:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878258.681280:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788878258.681282:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878258.681285:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878258.681287:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878258.681288:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878258.681289:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878258.681291:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878258.681294:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9197 02000000:00000001:3.0:1788878258.681294:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878258.681300:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878258.681301:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878258.681302:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878258.681303:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878258.681304:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878258.681305:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878258.681307:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878258.681309:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878258.681310:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878258.681312:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878258.681313:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878258.681314:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878258.681316:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878258.681317:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788878258.681318:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878258.681321:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878258.681321:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000008:00100000:2.0:1788878258.681324:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878258.681328:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681328:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878258.681332:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878258.681336:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878258.681337:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878258.681338:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878258.681339:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878258.681340:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878258.681344:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878258.681347:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290045568/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878258.681352:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878258.681354:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878258.681355:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878258.681357:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878258.681360:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878258.681361:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878258.681362:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878258.681364:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878258.681366:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9197 00000100:00080000:2.0:1788878258.681368:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878258.681369:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878258.681370:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878258.681373:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878258.681374:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878258.681375:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878258.681377:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878258.681378:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878258.681380:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:3.0:1788878258.681382:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788878258.681385:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878258.681388:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878258.681389:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878258.681391:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878258.681392:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878258.681394:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878258.681395:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878258.681397:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878258.681398:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878258.681401:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788878258.681401:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:2.0:1788878258.681404:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878258.681406:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878258.681408:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788878258.681408:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788878258.681409:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681412:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878258.681415:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681416:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:2.0:1788878258.681421:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290045696/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878258.681422:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681430:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303870505 00000100:00000001:2.0:1788878258.681431:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878258.681435:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878258.681437:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681446:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878258.681460:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303870505 00000100:00000001:3.0:1788878258.681465:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878258.681469:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878258.681471:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681473:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878258.681479:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878258/real 1788878258] req@ffff94e5045e6a00 x1875765290045312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878258.681490:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878258.681492:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878258.681496:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878258.681498:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878258.681500:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681502:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878258.681504:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878258.681508:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290045312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878258.681514:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681517:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878258.681519:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681520:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878258.681521:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681523:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878258.681525:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878258.681527:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878258.681530:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878258.681531:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878258.681534:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290045312:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878258.681539:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681541:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878258.681543:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878258.681545:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878258.681548:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878258.681550:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878258.681552:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878258.681556:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878258.681558:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878258.681561:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878258.681562:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681564:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681571:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878258.681572:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681574:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878258.681575:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878258.681576:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878258.681577:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878258.681578:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878258.681580:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878258.681581:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878258.681582:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878258.681585:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290045440:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878258.681587:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878258.681589:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878258.681590:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878258.681591:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878258.681593:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681597:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878258.681598:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:3.0:1788878258.681603:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878258.681611:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878258.681614:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878258.681615:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878258.681617:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681632:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878258.681635:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681639:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303869963 00000100:00000001:3.0:1788878258.681642:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303870505 00000100:00000001:3.0:1788878258.681644:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303869963 00000100:00000001:3.0:1788878258.681646:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303870505 00000100:00000001:3.0:1788878258.681647:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878258.681649:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681650:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878258.681653:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878258/real 1788878258] req@ffff94e5045e6680 x1875765290045440/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878258.681659:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878258.681660:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878258.681662:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878258.681663:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878258.681664:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681665:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878258.681666:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878258.681669:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290045440/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878258.681684:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681687:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878258.681689:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681691:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878258.681693:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681695:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878258.681715:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878258.681718:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878258.681722:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878258.681724:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878258.681727:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290045440:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878258.681730:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878258.681731:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878258.681732:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878258.681734:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878258.681735:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878258.681737:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878258.681739:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290045568:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878258.681741:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878258.681743:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878258.681744:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878258.681745:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878258.681748:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681753:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878258.681754:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:3.0:1788878258.681758:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878258.681768:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878258.681771:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878258.681773:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878258.681774:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681775:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878258.681777:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681780:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303869961 00000100:00000001:3.0:1788878258.681782:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303870505 00000100:00000001:3.0:1788878258.681784:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303869961 00000100:00000001:3.0:1788878258.681785:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303870505 00000100:00000001:3.0:1788878258.681787:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878258.681788:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681789:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878258.681791:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878258/real 1788878258] req@ffff94e5045e4a80 x1875765290045568/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878258.681796:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878258.681797:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878258.681798:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878258.681800:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878258.681801:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681802:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878258.681803:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878258.681806:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290045568/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878258.681810:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681811:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878258.681812:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681813:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878258.681814:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681815:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878258.681829:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878258.681831:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878258.681834:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878258.681835:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878258.681838:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290045568:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878258.681841:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878258.681842:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878258.681843:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878258.681845:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878258.681846:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878258.681847:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878258.681849:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290045696:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878258.681850:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878258.681851:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878258.681852:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878258.681853:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878258.681855:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681858:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878258.681859:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:3.0:1788878258.681862:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878258.681877:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878258.681880:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878258.681881:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878258.681882:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681884:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878258.681885:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681888:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303869961 00000100:00000001:3.0:1788878258.681890:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303870505 00000100:00000001:3.0:1788878258.681892:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303869961 00000100:00000001:3.0:1788878258.681894:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303870505 00000100:00000001:3.0:1788878258.681895:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878258.681896:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.681897:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878258.681900:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878258/real 1788878258] req@ffff94e5045e7100 x1875765290045696/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878258.681934:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878258.681935:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878258.681937:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878258.681938:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878258.681939:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681946:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878258.681947:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878258.681950:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290045696/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878258.681955:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681957:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878258.681959:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681960:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878258.681961:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681962:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878258.681975:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878258.681976:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878258.681978:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878258.681980:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878258.681982:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290045696:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878258.681985:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.681986:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878258.681988:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878258.681989:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878258.681991:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878258.681992:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878258.681993:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878258.681995:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878258.681995:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878258.681997:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878258.681998:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.682000:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878258.682001:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878258.682001:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878258.682003:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878258.682004:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878258.682005:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878258.682006:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878258.682006:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878258.682007:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878258.682008:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.682009:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878258.682010:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878258.682010:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878258.682011:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878258.682012:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878258.682013:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878258.682013:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878258.682014:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878258.682015:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878258.682016:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.682017:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878258.682019:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878258.682020:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878258.682021:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878258.682022:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878261.497135:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878261.497143:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878261.497149:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878263.801066:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878263.801083:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878263.801088:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878263.801094:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878263.801101:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878263.801113:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9202 00000100:00080000:2.0:1788878263.801131:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878263.801135:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878263.801138:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878263.801152:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878263.801156:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878263.801161:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878263.801165:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878263.801168:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878263.801170:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878263.801177:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878263.801181:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878263.801188:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878263.801196:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878263.801202:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801206:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878263.801210:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878263.801224:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878263.801226:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801231:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878263.801235:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878263.801242:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801266:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878263.801279:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290045824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878263.801292:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878263.801297:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801322:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878263.801328:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878263.801334:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878263.801336:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878263.801338:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878263.801344:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878263.801351:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9202 00000100:00080000:2.0:1788878263.801358:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878263.801362:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878263.801365:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878263.801372:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878263.801374:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878263.801377:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878263.801381:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878263.801385:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878263.801388:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878263.801395:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878263.801401:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878263.801405:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878263.801414:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878263.801418:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801424:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878263.801431:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878263.801437:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801441:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878263.801450:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878263.801463:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801465:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878263.801468:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878263.801472:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801482:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878263.801505:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290045952/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878263.801517:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878263.801521:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801525:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878263.801529:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878263.801533:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878263.801536:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878263.801538:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878263.801542:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878263.801545:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9202 00000100:00080000:2.0:1788878263.801549:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878263.801552:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878263.801554:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878263.801560:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878263.801562:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878263.801565:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878263.801568:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878263.801571:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878263.801573:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878263.801578:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878263.801581:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878263.801583:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878263.801587:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878263.801589:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801592:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878263.801635:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878263.801640:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801642:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878263.801648:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878263.801650:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801652:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878263.801654:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878263.801657:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801664:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878263.801670:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290046080/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878263.801690:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878263.801697:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801702:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878263.801706:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878263.801711:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878263.801713:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878263.801715:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878263.801719:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878263.801723:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9202 00000100:00080000:2.0:1788878263.801728:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878263.801731:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878263.801733:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878263.801739:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878263.801741:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878263.801744:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878263.801748:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878263.801751:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878263.801753:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878263.801757:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878263.801761:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878263.801763:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878263.801767:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878263.801769:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801772:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878263.801776:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878263.801779:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801782:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878263.801788:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878263.801790:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801792:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878263.801794:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878263.801797:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801805:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878263.801812:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290046208/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878263.801823:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878263.801827:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878263.801833:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878265.593078:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878265.593082:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878265.593096:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878268.921064:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878268.921079:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878268.921084:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878268.921089:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878268.921096:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878268.921107:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9207 00000100:00080000:2.0:1788878268.921122:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878268.921127:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878268.921130:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878268.921143:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878268.921149:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878268.921156:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878268.921162:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878268.921168:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878268.921171:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878268.921181:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878268.921188:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878268.921195:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878268.921204:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878268.921211:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.921217:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878268.921222:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878268.921243:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878268.921247:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.921253:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878268.921258:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878268.921277:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.921295:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878268.921327:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5f80 x1875765290046336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878268.921344:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878268.921352:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.921374:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878268.921380:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878268.921387:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878268.921390:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878268.921393:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878268.921398:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878268.921403:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9207 00000100:00080000:2.0:1788878268.921410:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878268.921414:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878268.921417:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878268.921423:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878268.921425:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878268.921429:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878268.921432:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878268.921436:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878268.921439:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878268.921443:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878268.921453:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878268.921456:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878268.921464:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878268.921467:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.921471:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878268.921477:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878268.921482:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.921485:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878268.921491:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878268.921495:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.921497:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878268.921499:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878268.921504:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.921522:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878268.921539:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4380 x1875765290046464/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878268.921559:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878268.921567:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.921646:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878268.921654:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878268.921662:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878268.921664:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878268.921667:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878268.921671:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878268.921677:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9207 00000100:00080000:2.0:1788878268.921685:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878268.921693:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878268.921699:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878268.921708:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878268.921710:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878268.921714:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878268.921717:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878268.921720:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878268.921723:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878268.921728:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878268.921730:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878268.921733:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878268.921738:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878268.921741:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.921744:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878268.921748:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878268.921751:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.921754:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878268.921769:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878268.921773:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.921777:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878268.921779:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878268.921784:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.921798:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878268.921816:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290046592/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878268.921841:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878268.921845:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.921850:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878268.921860:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878268.921865:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878268.921866:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878268.921868:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878268.921872:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878268.921875:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9207 00000100:00080000:2.0:1788878268.921881:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878268.921888:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878268.921891:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878268.921943:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878268.921947:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878268.921950:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878268.921954:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878268.921959:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878268.921962:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878268.921968:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878268.921974:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878268.921981:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878268.921985:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878268.921990:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.921994:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878268.921998:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878268.922008:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.922015:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878268.922025:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878268.922029:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.922037:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878268.922042:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878268.922047:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.922061:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878268.922076:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290046720/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878268.922092:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878268.922101:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878268.922106:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878269.689092:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878269.689097:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878269.689102:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1788878273.785057:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878273.785064:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878273.785069:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1788878277.881051:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878277.881055:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878277.881059:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878279.160997:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878279.161007:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878279.161010:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878279.161014:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878279.161019:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878279.161025:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9218 00000100:00080000:2.0:1788878279.161037:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878279.161040:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878279.161041:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878279.161050:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878279.161053:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878279.161056:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878279.161058:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878279.161060:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878279.161061:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878279.161065:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878279.161067:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878279.161070:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878279.161074:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878279.161076:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161078:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878279.161080:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878279.161099:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878279.161101:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161104:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878279.161106:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878279.161111:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161123:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878279.161133:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290047360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878279.161142:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878279.161146:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161159:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878279.161161:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878279.161164:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878279.161166:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878279.161167:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878279.161169:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878279.161171:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9218 00000100:00080000:2.0:1788878279.161173:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878279.161175:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878279.161176:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878279.161178:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878279.161179:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878279.161181:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878279.161182:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878279.161183:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878279.161185:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878279.161188:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878279.161190:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878279.161191:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878279.161198:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878279.161199:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161201:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878279.161205:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878279.161208:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161210:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878279.161213:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878279.161216:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161220:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878279.161222:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878279.161230:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161236:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878279.161244:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290047488/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878279.161258:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878279.161260:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161266:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878279.161270:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878279.161273:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878279.161275:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878279.161278:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878279.161283:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878279.161287:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9218 00000100:00080000:2.0:1788878279.161291:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878279.161294:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878279.161298:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878279.161305:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878279.161308:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878279.161311:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878279.161314:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878279.161316:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878279.161318:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878279.161322:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878279.161325:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878279.161328:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878279.161332:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878279.161334:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161336:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878279.161338:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878279.161344:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161349:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878279.161354:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878279.161356:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161359:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878279.161361:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878279.161366:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161373:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878279.161381:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290047616/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878279.161389:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878279.161393:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161395:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878279.161397:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878279.161399:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878279.161401:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878279.161402:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878279.161404:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878279.161409:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9218 00000100:00080000:2.0:1788878279.161413:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878279.161416:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878279.161417:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878279.161419:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878279.161420:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878279.161421:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878279.161424:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878279.161428:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878279.161429:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878279.161434:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878279.161438:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878279.161441:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878279.161445:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878279.161448:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161451:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878279.161455:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878279.161460:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161466:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878279.161471:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878279.161474:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161477:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878279.161478:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878279.161481:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161487:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878279.161496:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290047744/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878279.161523:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878279.161543:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878279.161551:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878281.977077:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878281.977080:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878281.977082:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878284.280957:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878284.280965:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878284.280967:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878284.280970:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878284.280974:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878284.280979:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9223 00000100:00080000:2.0:1788878284.280989:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878284.280990:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878284.280992:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878284.280998:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878284.281000:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878284.281003:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878284.281005:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878284.281007:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878284.281008:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878284.281011:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878284.281012:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878284.281015:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878284.281017:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878284.281019:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281020:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878284.281022:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878284.281031:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878284.281032:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281035:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878284.281037:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878284.281041:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281051:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878284.281058:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290047872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878284.281068:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878284.281074:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281101:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878284.281105:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878284.281108:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878284.281110:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878284.281112:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878284.281115:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878284.281118:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9223 00000100:00080000:2.0:1788878284.281122:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878284.281124:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878284.281126:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878284.281130:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878284.281132:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878284.281135:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878284.281137:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878284.281140:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878284.281144:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878284.281150:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878284.281156:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878284.281159:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878284.281164:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878284.281168:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281170:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878284.281173:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878284.281176:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281179:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878284.281183:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878284.281187:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281190:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878284.281192:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878284.281197:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281203:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878284.281211:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290048000/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878284.281217:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878284.281220:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281229:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878284.281234:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878284.281238:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878284.281240:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878284.281242:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878284.281245:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878284.281249:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9223 00000100:00080000:2.0:1788878284.281254:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878284.281256:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878284.281258:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878284.281262:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878284.281264:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878284.281267:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878284.281270:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878284.281276:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878284.281280:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878284.281286:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878284.281292:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878284.281298:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878284.281305:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878284.281313:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281319:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878284.281322:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878284.281327:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281332:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878284.281339:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878284.281343:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281347:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878284.281351:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878284.281356:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281366:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878284.281374:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290048128/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878284.281393:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878284.281397:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281401:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878284.281403:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878284.281411:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878284.281413:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878284.281416:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878284.281418:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878284.281420:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9223 00000100:00080000:2.0:1788878284.281424:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878284.281426:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878284.281427:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878284.281441:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878284.281444:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878284.281448:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878284.281452:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878284.281455:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878284.281457:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878284.281462:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878284.281466:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878284.281471:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878284.281474:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878284.281477:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281481:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878284.281485:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878284.281489:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281495:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878284.281499:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878284.281502:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281519:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878284.281522:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878284.281525:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281535:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878284.281542:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290048256/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878284.281551:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878284.281554:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878284.281559:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878286.073095:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878286.073100:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878286.073106:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878289.401061:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878289.401073:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878289.401076:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878289.401081:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878289.401086:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878289.401096:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9228 00000100:00080000:2.0:1788878289.401109:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878289.401112:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878289.401114:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878289.401123:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878289.401127:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878289.401132:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878289.401136:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878289.401138:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878289.401140:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878289.401144:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878289.401147:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878289.401167:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878289.401180:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878289.401183:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.401186:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878289.401189:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878289.401202:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878289.401205:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.401209:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878289.401213:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878289.401218:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.401232:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878289.401243:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290048384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878289.401256:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878289.401262:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.401290:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878289.401295:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878289.401299:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878289.401301:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878289.401303:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878289.401306:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878289.401309:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9228 00000100:00080000:2.0:1788878289.401314:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878289.401317:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878289.401318:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878289.401334:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878289.401335:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878289.401338:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878289.401340:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878289.401343:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878289.401344:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878289.401348:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878289.401351:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878289.401353:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878289.401360:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878289.401363:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.401366:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878289.401371:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878289.401376:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.401379:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878289.401384:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878289.401385:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.401387:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878289.401390:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878289.401398:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.401406:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878289.401438:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290048512/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878289.401460:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878289.401496:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.401512:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878289.401523:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878289.401534:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878289.401543:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878289.401551:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878289.401557:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878289.401567:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9228 00000100:00080000:2.0:1788878289.401577:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878289.401582:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878289.401585:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878289.401596:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878289.401602:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878289.401608:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878289.401616:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878289.401619:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878289.401622:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878289.401628:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878289.401632:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878289.401636:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878289.401642:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878289.401646:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.401648:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878289.401653:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878289.401656:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.401661:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878289.401673:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878289.401679:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.401683:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878289.401687:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878289.401691:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.401706:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878289.401723:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290048640/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878289.401748:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878289.401755:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.401784:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878289.401789:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878289.401797:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878289.401804:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878289.401810:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878289.401822:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878289.401834:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9228 00000100:00080000:2.0:1788878289.401848:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878289.401856:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878289.401861:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878289.401875:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878289.401878:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878289.401881:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878289.401887:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878289.401892:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878289.401897:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878289.401961:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878289.401971:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878289.401979:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878289.401989:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878289.401998:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.402006:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878289.402016:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878289.402025:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.402033:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878289.402045:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878289.402050:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.402058:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878289.402060:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878289.402064:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.402075:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878289.402092:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290048768/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878289.402125:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878289.402134:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878289.402144:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878290.169095:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878290.169099:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878290.169102:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1788878294.265090:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878294.265093:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878294.265096:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878294.521016:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878294.521028:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878294.521031:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878294.521035:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878294.521041:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878294.521050:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9233 00000100:00080000:2.0:1788878294.521066:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878294.521071:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878294.521074:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878294.521087:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878294.521091:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878294.521098:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878294.521103:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878294.521107:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878294.521110:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878294.521119:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878294.521139:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878294.521142:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878294.521145:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878294.521148:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521150:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878294.521153:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878294.521164:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878294.521166:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521169:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878294.521171:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878294.521175:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521187:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878294.521196:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4000 x1875765290048896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878294.521206:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878294.521210:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521224:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878294.521227:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878294.521231:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878294.521232:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878294.521234:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878294.521236:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878294.521239:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9233 00000100:00080000:2.0:1788878294.521243:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878294.521245:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878294.521246:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878294.521249:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878294.521251:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878294.521253:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878294.521255:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878294.521257:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878294.521259:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878294.521262:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878294.521264:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878294.521266:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878294.521271:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878294.521273:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521275:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878294.521279:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878294.521282:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521288:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878294.521292:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878294.521295:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521299:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878294.521303:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878294.521307:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521314:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878294.521327:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290049024/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878294.521347:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878294.521351:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521356:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878294.521361:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878294.521366:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878294.521386:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878294.521389:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878294.521390:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878294.521393:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9233 00000100:00080000:2.0:1788878294.521395:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878294.521397:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878294.521398:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878294.521400:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878294.521401:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878294.521405:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878294.521410:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878294.521412:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878294.521415:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878294.521417:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878294.521419:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878294.521420:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878294.521422:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878294.521424:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521425:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878294.521427:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878294.521428:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521429:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878294.521433:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878294.521435:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521436:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878294.521436:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878294.521438:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521442:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878294.521450:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290049152/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878294.521471:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878294.521473:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521478:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878294.521481:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878294.521488:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878294.521491:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878294.521493:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878294.521498:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878294.521502:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9233 00000100:00080000:2.0:1788878294.521507:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878294.521508:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878294.521510:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878294.521512:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878294.521515:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878294.521516:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878294.521519:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878294.521523:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878294.521525:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878294.521530:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878294.521534:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878294.521537:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878294.521542:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878294.521545:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521547:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878294.521550:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878294.521555:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521559:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878294.521564:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878294.521567:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521571:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878294.521574:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878294.521576:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521583:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878294.521590:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290049280/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878294.521600:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878294.521604:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878294.521608:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878298.361070:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878298.361074:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878298.361077:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788878299.641304:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878299.641313:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878299.641317:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878299.641323:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878299.641326:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878299.641337:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878299.641343:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878299.641365:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878299.641386:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290049408:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878299.641403:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878299.641430:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878299.641457:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878299.641467:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878299.641508:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.641534:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878299.641548:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:2.0:1788878299.641554:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878299.641617:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878299.641634:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878299.641642:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878299.641646:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.641648:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878299.641657:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.641664:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303911465 00000100:00000001:2.0:1788878299.641677:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878299.641687:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303911465 00000100:00000001:2.0:1788878299.641693:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878299.641701:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878299.641705:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.641710:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878299.641722:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878299/real 1788878299] req@ffff94e50897aa00 x1875765290049408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878299.641745:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878299.641747:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878299.641755:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878299.641759:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878299.641762:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.641770:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878299.641773:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878299.641781:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290049408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878299.641800:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.641811:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878299.641816:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.641820:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878299.641822:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.641833:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878299.641836:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878299.641841:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878299.641850:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878299.641855:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878299.641861:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290049408:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878299.641865:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.641867:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878299.641869:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878299.641874:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878299.641878:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878299.641882:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878299.641886:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878299.641891:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878299.641893:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878299.641898:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878299.641900:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.641906:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.641914:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878299.641946:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.641951:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878299.641953:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878299.641955:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878299.641957:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878299.641959:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878299.641962:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878299.641965:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878299.641967:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878299.641973:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290049536:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878299.641977:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878299.641980:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878299.641982:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878299.641984:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878299.641987:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.641996:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878299.641999:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00000001:2.0:1788878299.642006:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878299.642022:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878299.642029:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878299.642032:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878299.642038:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.642041:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878299.642048:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.642056:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303910924 00000100:00000001:2.0:1788878299.642061:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303911465 00000100:00000001:2.0:1788878299.642066:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303910924 00000100:00000001:2.0:1788878299.642071:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303911465 00000100:00000001:2.0:1788878299.642079:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878299.642082:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.642088:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878299.642093:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878299/real 1788878299] req@ffff94e508978700 x1875765290049536/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878299.642108:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878299.642111:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878299.642116:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878299.642118:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878299.642120:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642128:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878299.642131:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878299.642143:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290049536/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878299.642156:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642158:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878299.642163:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642174:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878299.642176:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642180:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878299.642203:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878299.642207:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878299.642212:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878299.642218:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878299.642227:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290049536:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878299.642241:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878299.642244:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878299.642247:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878299.642251:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878299.642256:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878299.642260:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878299.642268:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290049664:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878299.642272:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878299.642276:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878299.642279:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878299.642281:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878299.642285:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.642290:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878299.642291:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:2.0:1788878299.642294:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878299.642308:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878299.642312:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878299.642314:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878299.642315:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.642317:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878299.642318:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.642321:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303910922 00000100:00000001:2.0:1788878299.642323:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303911465 00000100:00000001:2.0:1788878299.642324:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303910922 00000100:00000001:2.0:1788878299.642326:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303911465 00000100:00000001:2.0:1788878299.642328:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878299.642329:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.642330:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878299.642332:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878299/real 1788878299] req@ffff94e50897a300 x1875765290049664/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878299.642338:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878299.642338:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878299.642340:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878299.642341:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878299.642342:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642343:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878299.642344:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878299.642346:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290049664/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878299.642351:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642352:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878299.642353:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642354:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878299.642355:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642356:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878299.642367:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878299.642369:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878299.642372:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878299.642373:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878299.642377:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290049664:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878299.642381:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642383:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878299.642384:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878299.642386:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878299.642388:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878299.642390:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878299.642391:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878299.642394:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878299.642395:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878299.642397:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878299.642398:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642400:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878299.642401:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878299.642403:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878299.642404:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878299.642405:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878299.642407:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878299.642409:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878299.642410:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878299.642412:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878299.642414:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642415:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642419:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878299.642420:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.642423:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878299.642424:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878299.642426:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878299.642427:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878299.642429:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878299.642432:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878299.642447:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878299.642449:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878299.642452:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290049792:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878299.642455:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878299.642457:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878299.642459:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878299.642460:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878299.642462:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.642467:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878299.642468:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:2.0:1788878299.642471:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878299.642480:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878299.642489:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878299.642491:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878299.642492:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.642493:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878299.642495:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.642498:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303910922 00000100:00000001:2.0:1788878299.642500:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303911465 00000100:00000001:2.0:1788878299.642502:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303910922 00000100:00000001:2.0:1788878299.642503:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303911465 00000100:00000001:2.0:1788878299.642504:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878299.642506:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.642507:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878299.642509:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878299/real 1788878299] req@ffff94e50897aa00 x1875765290049792/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878299.642514:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878299.642515:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878299.642516:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878299.642518:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878299.642519:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642520:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878299.642521:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878299.642524:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290049792/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878299.642529:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642530:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878299.642532:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642533:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878299.642534:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642536:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878299.642548:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878299.642550:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878299.642552:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878299.642554:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878299.642556:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290049792:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878299.642558:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642559:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878299.642560:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878299.642561:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878299.642562:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878299.642563:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878299.642564:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878299.642566:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878299.642567:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878299.642568:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878299.642569:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642570:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878299.642572:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878299.642573:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878299.642575:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878299.642576:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878302.457127:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878302.457134:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878302.457139:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878304.761050:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878304.761061:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878304.761064:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878304.761069:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878304.761074:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878304.761082:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9243 00000100:00080000:2.0:1788878304.761094:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878304.761097:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878304.761100:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878304.761108:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878304.761110:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878304.761114:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878304.761115:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878304.761117:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878304.761118:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878304.761122:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878304.761124:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878304.761126:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878304.761129:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878304.761132:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761134:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878304.761136:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878304.761148:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878304.761150:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761153:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878304.761155:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878304.761160:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761172:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878304.761181:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290049920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878304.761190:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878304.761195:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761213:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878304.761216:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878304.761219:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878304.761220:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878304.761221:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878304.761223:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878304.761225:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9243 00000100:00080000:2.0:1788878304.761227:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878304.761229:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878304.761230:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878304.761232:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878304.761233:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878304.761235:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878304.761236:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878304.761237:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878304.761238:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878304.761240:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878304.761242:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878304.761243:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878304.761249:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878304.761250:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761252:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878304.761256:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878304.761259:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761261:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878304.761263:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878304.761264:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761265:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878304.761267:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878304.761268:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761273:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878304.761285:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290050048/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878304.761294:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878304.761297:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761301:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878304.761304:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878304.761310:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878304.761311:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878304.761313:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878304.761318:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878304.761321:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9243 00000100:00080000:2.0:1788878304.761327:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878304.761331:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878304.761333:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878304.761338:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878304.761339:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878304.761342:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878304.761346:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878304.761354:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878304.761357:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878304.761362:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878304.761367:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878304.761369:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878304.761374:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878304.761375:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761377:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878304.761379:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878304.761383:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761385:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878304.761389:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878304.761390:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761392:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878304.761393:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878304.761394:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761398:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878304.761406:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290050176/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878304.761432:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878304.761435:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761437:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878304.761440:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878304.761443:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878304.761444:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878304.761446:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878304.761448:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878304.761450:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9243 00000100:00080000:2.0:1788878304.761454:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878304.761455:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878304.761456:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878304.761464:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878304.761465:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878304.761467:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878304.761470:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878304.761471:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878304.761472:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878304.761474:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878304.761476:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878304.761477:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878304.761479:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878304.761481:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761482:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878304.761485:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878304.761488:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761493:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878304.761497:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878304.761498:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761500:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878304.761501:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878304.761503:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761508:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878304.761514:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290050304/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878304.761533:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878304.761536:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878304.761538:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878306.553055:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878306.553061:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878306.553064:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1788878310.649099:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878310.649114:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878310.649118:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1788878314.745029:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878314.745033:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878314.745036:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1788878318.841202:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878318.841207:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878318.841212:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1788878322.937076:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878322.937080:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878322.937084:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1788878327.033159:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878327.033165:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878327.033170:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788878330.361460:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878330.361470:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878330.361477:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878330.361485:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878330.361491:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878330.361502:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878330.361510:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878330.361521:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878330.361536:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290052480:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878330.361549:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878330.361558:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878330.361565:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878330.361576:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878330.361587:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.361614:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878330.361620:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286517888 : -117763423033728 : ffff94e51100a680) 00000100:00000001:2.0:1788878330.361631:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878330.361709:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878330.361737:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878330.361768:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878330.361779:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.361790:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878330.361804:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.361819:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303942185 00000100:00000001:2.0:1788878330.361841:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878330.361858:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303942185 00000100:00000001:2.0:1788878330.361889:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878330.361898:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878330.361907:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.361914:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878330.361991:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878330/real 1788878330] req@ffff94e51100a680 x1875765290052480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878330.362011:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878330.362015:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878330.362021:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878330.362025:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878330.362028:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362049:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878330.362053:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878330.362063:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100a680 x1875765290052480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878330.362079:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362084:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878330.362091:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362094:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878330.362097:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362102:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878330.362106:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878330.362112:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878330.362119:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878330.362124:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878330.362132:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290052480:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878330.362141:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362145:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878330.362149:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878330.362153:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878330.362158:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878330.362161:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878330.362165:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878330.362171:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878330.362175:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878330.362180:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878330.362183:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362204:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362213:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878330.362215:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.362219:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878330.362221:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878330.362223:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878330.362226:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878330.362228:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878330.362232:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878330.362235:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878330.362238:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878330.362244:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290052608:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878330.362249:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878330.362253:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878330.362255:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878330.362258:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878330.362263:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.362272:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878330.362274:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:2.0:1788878330.362281:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878330.362353:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878330.362363:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878330.362368:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878330.362371:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.362375:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878330.362380:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.362388:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303941644 00000100:00000001:2.0:1788878330.362394:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303942185 00000100:00000001:2.0:1788878330.362399:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303941644 00000100:00000001:2.0:1788878330.362404:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303942185 00000100:00000001:2.0:1788878330.362408:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878330.362413:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.362417:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878330.362425:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878330/real 1788878330] req@ffff94e51100aa00 x1875765290052608/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878330.362444:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878330.362447:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878330.362453:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878330.362456:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878330.362459:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362462:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878330.362465:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878330.362473:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290052608/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878330.362482:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362486:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878330.362507:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362508:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878330.362510:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362511:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878330.362523:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878330.362525:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878330.362528:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878330.362531:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878330.362534:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290052608:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878330.362538:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878330.362540:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878330.362542:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878330.362544:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878330.362546:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878330.362548:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878330.362551:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290052736:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878330.362554:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878330.362556:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878330.362559:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878330.362560:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878330.362564:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.362570:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878330.362571:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:2.0:1788878330.362576:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878330.362590:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878330.362595:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878330.362597:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878330.362599:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.362600:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878330.362603:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.362606:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303941642 00000100:00000001:2.0:1788878330.362609:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303942185 00000100:00000001:2.0:1788878330.362611:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303941642 00000100:00000001:2.0:1788878330.362613:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303942185 00000100:00000001:2.0:1788878330.362616:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878330.362617:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.362619:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878330.362623:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878330/real 1788878330] req@ffff94e51100b480 x1875765290052736/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878330.362630:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878330.362632:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878330.362634:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878330.362635:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878330.362637:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362639:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878330.362640:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878330.362644:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290052736/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878330.362651:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362653:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878330.362656:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362657:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878330.362659:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362661:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878330.362671:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878330.362674:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878330.362677:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878330.362680:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878330.362683:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290052736:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878330.362686:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878330.362687:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878330.362688:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878330.362690:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878330.362692:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878330.362694:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878330.362696:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290052864:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878330.362700:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878330.362701:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878330.362703:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878330.362704:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878330.362707:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.362711:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878330.362712:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:2.0:1788878330.362716:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878330.362743:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878330.362748:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878330.362750:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878330.362752:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.362754:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878330.362758:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.362761:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303941642 00000100:00000001:2.0:1788878330.362764:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303942185 00000100:00000001:2.0:1788878330.362768:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303941642 00000100:00000001:2.0:1788878330.362771:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303942185 00000100:00000001:2.0:1788878330.362773:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878330.362776:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.362778:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878330.362782:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878330/real 1788878330] req@ffff94e51100b800 x1875765290052864/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878330.362791:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878330.362793:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878330.362795:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878330.362797:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878330.362798:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362801:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878330.362802:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878330.362806:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290052864/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878330.362815:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362818:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878330.362820:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362823:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878330.362824:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362826:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878330.362837:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878330.362840:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878330.362843:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878330.362846:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878330.362850:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290052864:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878330.362854:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362857:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878330.362859:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878330.362861:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878330.362864:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878330.362866:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878330.362867:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878330.362870:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878330.362872:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878330.362875:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878330.362877:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362879:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878330.362880:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878330.362899:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878330.362902:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878330.362903:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878330.362904:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878330.362906:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878330.362907:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878330.362908:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878330.362909:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362911:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878330.362912:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878330.362913:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878330.362914:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878330.362915:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878330.362916:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878330.362918:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878330.362919:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878330.362920:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878330.362921:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362923:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878330.362927:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878330.362928:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878330.362930:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878330.362932:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878331.129128:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878331.129133:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878331.129137:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1788878335.225062:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878335.225066:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878335.225069:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1788878339.321160:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878339.321179:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878339.321181:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788878340.601236:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878340.601248:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878340.601265:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878340.601269:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878340.601273:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878340.601285:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878340.601292:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878340.601301:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878340.601322:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290053504:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878340.601339:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878340.601347:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878340.601354:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878340.601366:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878340.601393:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.601463:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878340.601469:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:2.0:1788878340.601478:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878340.601545:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878340.601565:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878340.601576:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878340.601583:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.601588:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878340.601595:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.601605:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303952425 00000100:00000001:2.0:1788878340.601629:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878340.601638:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303952425 00000100:00000001:2.0:1788878340.601644:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878340.601651:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878340.601656:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.601659:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878340.601669:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878340/real 1788878340] req@ffff94e50897aa00 x1875765290053504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878340.601685:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878340.601687:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878340.601693:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878340.601696:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878340.601698:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.601701:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878340.601721:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878340.601726:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290053504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878340.601736:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.601739:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878340.601741:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.601743:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878340.601743:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.601745:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878340.601747:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878340.601749:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878340.601752:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878340.601754:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878340.601756:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290053504:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878340.601761:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.601762:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878340.601764:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878340.601766:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878340.601771:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878340.601773:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878340.601775:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878340.601779:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878340.601780:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878340.601784:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878340.601785:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.601786:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.601793:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878340.601795:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.601797:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878340.601798:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878340.601799:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878340.601801:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878340.601802:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878340.601804:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878340.601805:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878340.601807:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878340.601810:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290053632:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878340.601812:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878340.601815:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878340.601816:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878340.601818:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878340.601821:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.601829:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878340.601830:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:2.0:1788878340.601835:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878340.601850:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878340.601855:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878340.601858:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878340.601860:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.601862:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878340.601865:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.601870:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303951883 00000100:00000001:2.0:1788878340.601872:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303952425 00000100:00000001:2.0:1788878340.601874:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303951883 00000100:00000001:2.0:1788878340.601876:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303952425 00000100:00000001:2.0:1788878340.601878:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878340.601879:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.601880:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878340.601884:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878340/real 1788878340] req@ffff94e50897a300 x1875765290053632/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878340.601893:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878340.601895:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878340.601898:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878340.601900:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878340.601901:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.601903:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878340.601904:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878340.601909:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290053632/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878340.601948:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.601950:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878340.601951:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.601953:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878340.601954:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.601955:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878340.601981:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878340.601983:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878340.601986:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878340.601988:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878340.601991:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290053632:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878340.601994:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878340.601995:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878340.601996:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878340.601998:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878340.602000:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878340.602001:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878340.602003:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290053760:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878340.602005:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878340.602007:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878340.602008:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878340.602010:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878340.602012:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.602018:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878340.602019:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00000001:2.0:1788878340.602022:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878340.602032:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878340.602036:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878340.602037:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878340.602038:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.602040:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878340.602042:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.602044:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303951882 00000100:00000001:2.0:1788878340.602046:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303952425 00000100:00000001:2.0:1788878340.602048:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303951882 00000100:00000001:2.0:1788878340.602049:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303952425 00000100:00000001:2.0:1788878340.602051:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878340.602052:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.602053:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878340.602056:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878340/real 1788878340] req@ffff94e508978700 x1875765290053760/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878340.602061:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878340.602062:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878340.602064:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878340.602065:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878340.602066:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.602067:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878340.602068:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878340.602071:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290053760/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878340.602076:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.602077:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878340.602079:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.602080:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878340.602081:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.602082:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878340.602093:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878340.602095:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878340.602097:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878340.602098:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878340.602100:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290053760:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878340.602119:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878340.602120:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878340.602121:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878340.602123:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878340.602124:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878340.602126:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878340.602128:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290053888:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878340.602131:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878340.602133:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878340.602134:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878340.602135:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878340.602137:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.602141:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878340.602142:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:2.0:1788878340.602145:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878340.602153:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878340.602155:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878340.602157:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878340.602158:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.602160:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878340.602162:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.602165:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303951882 00000100:00000001:2.0:1788878340.602167:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303952425 00000100:00000001:2.0:1788878340.602168:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303951882 00000100:00000001:2.0:1788878340.602170:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303952425 00000100:00000001:2.0:1788878340.602172:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878340.602173:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.602174:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878340.602177:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878340/real 1788878340] req@ffff94e508978a80 x1875765290053888/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878340.602189:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878340.602190:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878340.602191:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878340.602193:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878340.602193:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.602195:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878340.602196:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878340.602198:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290053888/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878340.602202:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.602204:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878340.602205:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.602206:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878340.602207:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.602209:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878340.602221:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878340.602222:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878340.602224:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878340.602226:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878340.602229:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290053888:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878340.602232:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.602234:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878340.602234:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878340.602236:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878340.602237:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878340.602238:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878340.602239:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878340.602241:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878340.602242:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878340.602244:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878340.602245:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.602247:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878340.602248:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878340.602262:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878340.602264:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878340.602265:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878340.602266:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878340.602268:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878340.602269:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878340.602270:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878340.602271:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.602273:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878340.602273:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878340.602274:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878340.602276:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878340.602277:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878340.602278:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878340.602279:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878340.602280:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878340.602281:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878340.602282:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.602283:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878340.602286:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878340.602287:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878340.602304:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878340.602305:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878343.417054:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878343.417063:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878343.417066:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878345.721040:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878345.721053:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878345.721057:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878345.721061:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878345.721067:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878345.721075:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9284 00000100:00080000:3.0:1788878345.721088:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878345.721092:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878345.721095:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878345.721105:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878345.721109:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878345.721113:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878345.721116:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878345.721119:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878345.721121:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878345.721151:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878345.721154:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878345.721158:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878345.721162:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878345.721166:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878345.721169:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878345.721173:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878345.721185:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878345.721187:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878345.721191:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878345.721195:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878345.721200:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878345.721216:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878345.721246:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290054016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878345.721271:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878345.721277:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878345.721295:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878345.721299:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878345.721304:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878345.721306:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878345.721309:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878345.721312:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878345.721316:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9284 00000100:00080000:3.0:1788878345.721326:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878345.721328:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878345.721330:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878345.721334:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878345.721336:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878345.721339:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878345.721341:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878345.721344:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878345.721346:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878345.721350:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878345.721354:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878345.721356:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878345.721364:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878345.721368:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878345.721371:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878345.721376:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878345.721380:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878345.721383:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.721387:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788878345.721389:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878345.721396:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.721398:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878345.721400:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878345.721407:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878345.721408:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788878345.721411:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.721414:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878345.721419:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878345.721422:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878345.721427:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1788878345.721436:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290054144/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788878345.721437:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878345.721447:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878345.721466:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1788878345.721467:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290054016:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878345.721473:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.721482:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788878345.721485:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878345.721489:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1788878345.721492:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1788878345.721494:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878345.721507:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788878345.721507:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878345.721514:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878345.721516:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1788878345.721518:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878345.721521:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878345.721539:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9284 00000100:00000001:2.0:1788878345.721544:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788878345.721545:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878345.721553:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:3.0:1788878345.721554:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878345.721559:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878345.721565:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1788878345.721567:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878345.721571:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878345.721574:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878345.721578:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878345.721580:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878345.721582:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878345.721587:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878345.721592:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878345.721594:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878345.721599:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878345.721603:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878345.721605:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878345.721608:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878345.721611:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878345.721614:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788878345.721622:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788878345.721627:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878345.721632:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878345.721636:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878345.721638:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878345.721641:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788878345.721642:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.721652:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878345.721654:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.721659:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.721668:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:3.0:1788878345.721669:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290054272/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878345.721675:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.721688:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303957545 00000100:00000001:3.0:1788878345.721701:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878345.721704:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878345.721710:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878345.721718:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878345.721721:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878345.721725:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878345.721728:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788878345.721731:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303957545 00000100:00080000:3.0:1788878345.721733:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878345.721743:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878345.721747:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788878345.721752:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788878345.721760:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9284 00000100:00000001:2.0:1788878345.721761:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.721769:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788878345.721776:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788878345.721784:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878345/real 1788878345] req@ffff94e50449c000 x1875765290054016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878345.721800:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000400:00000001:2.0:1788878345.721803:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788878345.721809:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:2.0:1788878345.721810:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878345.721818:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000020:00000001:3.0:1788878345.721824:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788878345.721825:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000020:00000001:3.0:1788878345.721834:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788878345.721835:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788878345.721843:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878345.721844:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878345.721852:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788878345.721853:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878345.721860:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1788878345.721863:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290054016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878345.721864:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878345.721878:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878345.721889:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.721896:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878345.721897:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878345.721946:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878345.721955:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.721956:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878345.721963:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878345.721965:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1788878345.721971:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878345.721972:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878345.721976:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1788878345.721984:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.721985:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878345.721990:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878345.721995:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878345.721998:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1788878345.722010:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00080000:2.0:1788878345.722011:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 02000000:00000001:3.0:1788878345.722020:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722021:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788878345.722032:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1788878345.722036:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290054016:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788878345.722038:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878345.722046:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722049:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722055:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878345.722057:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722059:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878345.722066:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878345.722072:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:3.0:1788878345.722075:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290054400/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:2.0:1788878345.722077:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878345.722085:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878345.722093:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1788878345.722097:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788878345.722098:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878345.722107:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722108:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878345.722115:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878345.722117:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722121:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722142:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878345.722145:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722148:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878345.722151:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878345.722153:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878345.722157:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878345.722159:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878345.722163:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878345.722166:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878345.722169:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878345.722175:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290054144:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878345.722179:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878345.722181:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878345.722183:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878345.722185:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878345.722190:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722197:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878345.722199:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:2.0:1788878345.722205:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878345.722222:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878345.722264:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878345.722269:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878345.722273:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722286:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878345.722292:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722299:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303957004 00000100:00000001:2.0:1788878345.722305:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303957545 00000100:00000001:2.0:1788878345.722310:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303957004 00000100:00000001:2.0:1788878345.722315:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303957545 00000100:00000001:2.0:1788878345.722319:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878345.722323:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722327:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878345.722337:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878345/real 1788878345] req@ffff94e50449d180 x1875765290054144/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878345.722354:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878345.722356:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878345.722360:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878345.722363:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878345.722365:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722367:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878345.722369:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878345.722374:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290054144/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878345.722383:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722386:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878345.722389:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722391:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878345.722393:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722395:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878345.722412:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878345.722415:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878345.722420:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878345.722422:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878345.722427:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290054144:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878345.722431:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878345.722433:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878345.722435:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878345.722437:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878345.722440:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878345.722443:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878345.722446:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290054272:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878345.722450:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878345.722453:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878345.722455:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878345.722457:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878345.722461:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722468:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878345.722470:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:2.0:1788878345.722474:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878345.722492:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878345.722498:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878345.722501:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878345.722503:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722505:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878345.722508:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722512:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303957002 00000100:00000001:2.0:1788878345.722515:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303957545 00000100:00000001:2.0:1788878345.722518:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303957002 00000100:00000001:2.0:1788878345.722521:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303957545 00000100:00000001:2.0:1788878345.722524:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878345.722526:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722528:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878345.722534:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878345/real 1788878345] req@ffff94e50449f800 x1875765290054272/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878345.722544:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878345.722545:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878345.722548:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878345.722550:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878345.722552:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722554:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878345.722556:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878345.722560:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290054272/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878345.722568:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722570:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878345.722573:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722575:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878345.722576:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722578:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878345.722592:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878345.722595:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878345.722598:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878345.722601:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878345.722605:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290054272:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878345.722609:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878345.722611:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878345.722613:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878345.722615:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878345.722617:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878345.722619:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878345.722622:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290054400:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878345.722626:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878345.722628:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878345.722630:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878345.722632:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878345.722636:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722641:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878345.722642:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:2.0:1788878345.722646:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878345.722657:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878345.722661:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878345.722663:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878345.722665:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722667:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878345.722670:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722673:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303957002 00000100:00000001:2.0:1788878345.722677:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303957545 00000100:00000001:2.0:1788878345.722679:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303957002 00000100:00000001:2.0:1788878345.722682:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303957545 00000100:00000001:2.0:1788878345.722684:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878345.722687:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722689:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878345.722693:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878345/real 1788878345] req@ffff94e50449ca80 x1875765290054400/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878345.722702:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878345.722703:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878345.722706:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878345.722707:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878345.722709:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722711:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878345.722713:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878345.722717:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290054400/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878345.722725:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722728:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878345.722731:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722733:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878345.722735:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722737:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878345.722750:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878345.722753:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878345.722757:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878345.722760:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878345.722764:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290054400:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878345.722767:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722770:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878345.722771:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878345.722773:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878345.722776:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878345.722777:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878345.722779:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878345.722781:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878345.722783:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878345.722785:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878345.722787:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722789:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878345.722791:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878345.722792:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878345.722795:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878345.722797:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878345.722798:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878345.722800:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878345.722801:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878345.722803:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878345.722805:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722806:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878345.722808:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878345.722809:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878345.722811:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878345.722812:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878345.722813:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878345.722815:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878345.722817:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878345.722819:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878345.722821:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722823:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878345.722827:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878345.722829:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878345.722832:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878345.722834:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878347.513216:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878347.513235:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878347.513240:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:2.0:1788878350.841488:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878350.841498:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878350.841504:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878350.841512:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878350.841519:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878350.841533:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878350.841541:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878350.841548:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878350.841564:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290054528:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878350.841578:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878350.841588:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878350.841594:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878350.841602:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878350.841613:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878350.841654:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878350.841659:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:2.0:1788878350.841667:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878350.841746:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878350.841760:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303962665 00000100:00000001:2.0:1788878350.841781:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878350.841811:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303962665 00000100:00000001:2.0:1788878350.841823:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878350.841834:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878350.841841:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878350.841851:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878350.841857:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878350.841871:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878350.841880:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788878350.841892:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878350.841899:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878350.841961:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878350.841964:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878350.841967:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878350.841971:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878350.841974:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878350.841977:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878350.841997:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290054656:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878350.842007:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878350.842015:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878350.842021:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878350.842028:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878350.842036:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878350.842060:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878350.842066:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:2.0:1788878350.842079:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878350.842101:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878350.842132:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303962124 00000100:00000001:2.0:1788878350.842137:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303962665 00000100:00000001:2.0:1788878350.842141:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303962124 00000100:00000001:2.0:1788878350.842145:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303962665 00000100:00000001:2.0:1788878350.842148:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878350.842151:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878350.842155:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878350.842158:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878350.842160:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878350.842166:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878350.842169:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878350.842173:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878350.842179:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290054784:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878350.842184:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878350.842187:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878350.842190:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878350.842193:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878350.842196:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878350.842224:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878350.842227:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:2.0:1788878350.842232:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878350.842249:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878350.842253:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303962122 00000100:00000001:2.0:1788878350.842265:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303962665 00000100:00000001:2.0:1788878350.842268:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303962122 00000100:00000001:2.0:1788878350.842271:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303962665 00000100:00000001:2.0:1788878350.842273:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878350.842276:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878350.842281:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878350.842283:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878350.842287:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878350.842288:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788878350.842291:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878350.842292:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878350.842294:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878350.842295:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878350.842297:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878350.842299:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878350.842301:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878350.842304:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878350.842308:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290054912:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878350.842312:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878350.842315:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878350.842318:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878350.842320:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878350.842326:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878350.842331:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878350.842333:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:2.0:1788878350.842336:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878350.842348:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878350.842352:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303962122 00000100:00000001:2.0:1788878350.842356:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303962665 00000100:00000001:2.0:1788878350.842359:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303962122 00000100:00000001:2.0:1788878350.842362:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303962665 00000100:00000001:2.0:1788878350.842364:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878350.842367:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878350.842370:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878350.842373:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878350.842376:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878350.842378:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788878350.842383:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878350.842384:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878350.842386:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878350.842391:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878351.609168:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878351.609174:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878351.609179:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788878353.913364:0:2351:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1788878353.913417:0:2351:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788878353.913438:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788878353.913457:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878353.913478:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878353.913481:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878353.913485:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:3.0:1788878353.913496:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788878353.913502:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878353.913505:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878353.913508:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878353.913511:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:3.0:1788878353.913517:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788878353.913524:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878353.913527:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878353.913529:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878353.913531:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:3.0:1788878353.913537:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788878353.913543:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878353.913546:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878353.913548:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878353.913550:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878353.913562:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878353.913571:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878353.913577:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878353.913599:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878350/real 1788878353] req@ffff94e508978e00 x1875765290054912/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878353.913625:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878353.913629:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878353.913641:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878353.913645:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878353.913649:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.913654:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878353.913658:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878353.913669:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290054912/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878353.913686:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.913692:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878353.913698:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.913701:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878353.913705:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.913710:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878353.913764:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878353.913771:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878353.913781:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878353.913787:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878353.913794:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290054912:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878353.913803:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878353.913811:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878350/real 1788878353] req@ffff94e50897bb80 x1875765290054656/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878353.913830:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878353.913832:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878353.913838:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878353.913842:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878353.913844:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.913848:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878353.913851:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878353.913859:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290054656/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878353.913872:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.913876:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878353.913880:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.913883:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878353.913885:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.913888:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878353.913964:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878353.913970:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878353.913977:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878353.913982:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878353.913989:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290054656:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878353.914013:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878353.914019:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878350/real 1788878353] req@ffff94e50897ad80 x1875765290054784/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878353.914033:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878353.914035:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878353.914040:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878353.914043:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878353.914045:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.914048:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878353.914050:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878353.914057:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290054784/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878353.914069:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.914073:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878353.914077:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.914079:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878353.914081:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.914084:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878353.914103:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878353.914107:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878353.914113:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878353.914117:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878353.914123:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290054784:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878353.914131:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878353.914138:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878350/real 1788878353] req@ffff94e508979c00 x1875765290054528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878353.914152:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878353.914154:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878353.914160:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878353.914163:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878353.914166:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.914169:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878353.914171:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878353.914178:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290054528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878353.914213:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.914218:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878353.914222:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.914225:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878353.914227:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.914230:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878353.914232:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878353.914237:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878353.914243:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878353.914247:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878353.914253:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290054528:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878353.914260:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.914264:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878353.914266:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878353.914271:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878353.914279:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878353.914283:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878353.914286:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878353.914292:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878353.914296:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878353.914302:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878353.914304:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.914308:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878353.914311:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878353.914316:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878353.914321:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878353.914324:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878353.914326:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878353.914331:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878353.914334:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878353.914339:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878353.914342:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.914346:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878353.914348:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878353.914351:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878353.914356:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878353.914359:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878353.914362:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878353.914366:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878353.914369:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878353.914374:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878353.914377:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.914381:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878353.914383:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878353.914386:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878353.914400:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878353.914402:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878353.914404:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878353.914407:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878353.914410:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878353.914412:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878353.914414:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.914418:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878353.914429:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878353.914433:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878353.914436:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878353.914440:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878355.961117:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878355.961130:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878355.961133:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878355.961138:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878355.961143:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878355.961152:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9294 00000100:00080000:3.0:1788878355.961166:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878355.961169:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878355.961171:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878355.961198:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878355.961201:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878355.961206:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878355.961209:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878355.961211:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878355.961213:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878355.961218:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878355.961222:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878355.961226:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878355.961265:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878355.961269:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878355.961272:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878355.961276:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878355.961286:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878355.961288:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878355.961292:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878355.961296:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878355.961302:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878355.961315:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878355.961327:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290055040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878355.961343:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878355.961351:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878355.961377:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878355.961384:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878355.961391:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878355.961394:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878355.961397:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878355.961419:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878355.961424:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9294 00000100:00080000:3.0:1788878355.961432:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878355.961436:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878355.961439:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878355.961444:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878355.961448:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878355.961452:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878355.961457:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878355.961461:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878355.961465:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878355.961471:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878355.961477:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878355.961481:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878355.961491:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788878355.961492:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878355.961503:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788878355.961503:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878355.961511:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878355.961513:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878355.961521:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1788878355.961523:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878355.961525:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878355.961549:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1788878355.961549:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878355.961559:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878355.961559:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878355.961570:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878355.961572:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878355.961579:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878355.961583:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290055040:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878355.961586:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878355.961591:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878355.961599:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.961611:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878355.961619:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878355.961623:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878355.961626:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878355.961634:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1788878355.961642:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290055168/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788878355.961647:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878355.961675:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878355.961676:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878355.961683:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.961685:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:2.0:1788878355.961696:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878355.961698:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878355.961707:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878355.961714:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878355.961717:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878355.961721:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878355.961727:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878355.961732:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9294 00000100:00080000:3.0:1788878355.961740:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878355.961744:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878355.961746:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878355.961753:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000100:2.0:1788878355.961758:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:3.0:1788878355.961759:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878355.961768:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878355.961773:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878355.961778:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878355.961780:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878355.961782:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:3.0:1788878355.961791:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878355.961793:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878355.961800:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878355.961802:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.961809:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788878355.961812:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878355.961822:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878355.961827:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878355.961830:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878355.961834:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878355.961839:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.961841:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878355.961843:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.961855:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303967785 02000000:00000001:3.0:1788878355.961860:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878355.961865:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878355.961869:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878355.961872:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878355.961875:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788878355.961877:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878355.961894:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878355.961950:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290055296/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878355.961966:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878355.961972:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303967786 00000100:00000001:3.0:1788878355.961974:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878355.961982:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878355.961990:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878355.961995:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878355.961998:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788878355.962000:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788878355.962003:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878355.962012:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878355.962013:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788878355.962022:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878355.962023:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9294 00000100:00000001:2.0:1788878355.962032:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788878355.962038:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878355.962048:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1788878355.962051:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878355/real 1788878355] req@ffff94e50449f100 x1875765290055040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878355.962053:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878355.962067:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878355.962073:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878355.962081:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000400:00000001:2.0:1788878355.962086:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1788878355.962089:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000400:00000001:2.0:1788878355.962093:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878355.962098:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878355.962105:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878355.962106:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878355.962114:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00080000:3.0:1788878355.962121:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878355.962123:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.962133:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878355.962135:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.962139:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878355.962143:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878355.962154:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1788878355.962155:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290055040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788878355.962165:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878355.962174:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878355.962205:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878355.962220:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1788878355.962221:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878355.962233:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.962234:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878355.962244:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878355.962246:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878355.962249:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788878355.962253:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.962256:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878355.962261:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878355.962264:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:3.0:1788878355.962269:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878355.962272:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878355.962276:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878355.962281:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788878355.962292:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878355.962294:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878355.962306:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878355.962318:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290055040:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1788878355.962323:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290055424/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878355.962331:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.962341:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878355.962347:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1788878355.962353:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788878355.962355:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788878355.962365:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878355.962366:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878355.962376:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:3.0:1788878355.962376:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878355.962381:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878355.962391:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878355.962394:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878355.962401:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878355.962403:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.962408:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.962421:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878355.962425:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.962430:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878355.962433:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878355.962436:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878355.962439:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878355.962442:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878355.962446:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878355.962449:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878355.962453:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878355.962460:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290055168:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878355.962467:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878355.962471:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878355.962474:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878355.962477:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878355.962484:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.962496:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878355.962499:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:2.0:1788878355.962506:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878355.962534:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878355.962543:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878355.962548:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878355.962551:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.962554:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878355.962560:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.962567:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303967244 00000100:00000001:2.0:1788878355.962572:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303967785 00000100:00000001:2.0:1788878355.962577:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303967244 00000100:00000001:2.0:1788878355.962581:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303967786 00000100:00000001:2.0:1788878355.962584:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878355.962588:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.962592:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878355.962600:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878355/real 1788878355] req@ffff94e50449e300 x1875765290055168/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878355.962617:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878355.962619:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878355.962624:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878355.962628:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878355.962630:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.962633:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878355.962636:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878355.962644:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290055168/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878355.962659:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.962663:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878355.962668:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.962671:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878355.962675:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.962678:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878355.962700:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878355.962706:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878355.962713:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878355.962716:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878355.962720:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290055168:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878355.962725:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878355.962728:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878355.962730:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878355.962733:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878355.962735:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878355.962738:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878355.962741:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290055296:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878355.962745:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878355.962748:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878355.962751:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878355.962755:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878355.962760:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.962771:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878355.962774:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:2.0:1788878355.962781:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878355.962805:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878355.962822:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878355.962828:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878355.962830:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.962833:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878355.962838:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.962844:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303967242 00000100:00000001:2.0:1788878355.962848:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303967785 00000100:00000001:2.0:1788878355.962852:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303967242 00000100:00000001:2.0:1788878355.962856:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303967786 00000100:00000001:2.0:1788878355.962860:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878355.962863:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.962867:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878355.962875:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878355/real 1788878355] req@ffff94e50449ca80 x1875765290055296/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878355.962892:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878355.962894:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878355.962900:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878355.962943:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878355.962947:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.962952:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878355.962955:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878355.962964:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290055296/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878355.962978:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.962982:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878355.962987:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.962991:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878355.962993:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.962997:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878355.963016:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878355.963022:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878355.963028:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878355.963033:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878355.963039:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290055296:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878355.963046:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878355.963048:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878355.963051:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878355.963055:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878355.963058:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878355.963062:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878355.963068:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290055424:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878355.963074:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878355.963077:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878355.963079:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878355.963082:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878355.963088:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.963098:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878355.963100:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:2.0:1788878355.963106:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878355.963130:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878355.963151:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878355.963156:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878355.963159:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.963162:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878355.963168:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.963176:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303967243 00000100:00000001:2.0:1788878355.963204:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303967785 00000100:00000001:2.0:1788878355.963210:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303967243 00000100:00000001:2.0:1788878355.963216:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303967786 00000100:00000001:2.0:1788878355.963221:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878355.963226:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.963229:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878355.963238:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878355/real 1788878355] req@ffff94e50449f800 x1875765290055424/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878355.963250:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878355.963251:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878355.963256:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878355.963259:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878355.963262:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.963265:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878355.963268:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878355.963276:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290055424/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878355.963301:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.963305:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878355.963309:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.963312:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878355.963316:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.963319:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878355.963338:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878355.963343:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878355.963350:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878355.963354:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878355.963360:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290055424:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878355.963367:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.963370:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878355.963373:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878355.963376:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878355.963382:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878355.963385:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878355.963388:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878355.963392:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878355.963395:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878355.963398:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878355.963401:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.963404:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878355.963406:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878355.963408:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878355.963411:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878355.963413:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878355.963416:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878355.963419:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878355.963421:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878355.963424:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878355.963426:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.963429:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878355.963431:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878355.963434:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878355.963437:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878355.963439:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878355.963441:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878355.963444:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878355.963446:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878355.963449:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878355.963451:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.963455:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878355.963462:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878355.963465:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878355.963469:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878355.963472:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878361.081013:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878361.081025:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878361.081028:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878361.081033:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878361.081039:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878361.081048:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9299 00000100:00080000:3.0:1788878361.081063:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878361.081067:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878361.081071:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878361.081096:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878361.081101:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878361.081107:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878361.081112:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878361.081116:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878361.081119:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878361.081128:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878361.081134:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878361.081140:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878361.081146:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878361.081151:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878361.081156:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878361.081184:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878361.081212:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878361.081216:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878361.081222:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878361.081227:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878361.081234:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878361.081251:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878361.081276:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290055552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878361.081290:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878361.081295:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878361.081316:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878361.081321:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878361.081325:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878361.081327:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878361.081330:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878361.081335:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878361.081338:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9299 00000100:00080000:3.0:1788878361.081345:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878361.081348:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878361.081351:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878361.081356:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878361.081359:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878361.081363:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878361.081367:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878361.081370:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878361.081372:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878361.081376:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878361.081379:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878361.081381:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878361.081389:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878361.081392:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.081395:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878361.081396:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878361.081405:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00100000:3.0:1788878361.081406:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878361.081412:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00000001:3.0:1788878361.081414:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.081418:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878361.081420:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.081423:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878361.081428:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878361.081433:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878361.081434:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878361.081442:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878361.081442:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878361.081448:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878361.081452:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878361.081454:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878361.081463:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878361.081464:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290055552:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878361.081476:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1788878361.081478:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290055680/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788878361.081483:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878361.081489:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878361.081498:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878361.081502:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878361.081507:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878361.081508:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878361.081513:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878361.081520:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878361.081524:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878361.081526:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878361.081528:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878361.081532:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878361.081533:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788878361.081539:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00080000:3.0:1788878361.081544:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9299 00000100:00000001:2.0:1788878361.081548:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788878361.081552:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878361.081558:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878361.081562:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878361.081584:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878361.081588:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878361.081591:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878361.081596:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000800:00000100:2.0:1788878361.081605:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878361.081608:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878361.081616:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878361.081625:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:3.0:1788878361.081629:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878361.081634:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878361.081642:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878361.081651:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878361.081657:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878361.081658:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878361.081669:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.081671:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878361.081672:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878361.081691:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878361.081693:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788878361.081695:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878361.081704:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.081709:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303972905 02000000:00000001:3.0:1788878361.081712:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878361.081720:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878361.081725:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878361.081728:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878361.081733:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788878361.081734:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878361.081754:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878361.081766:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290055808/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878361.081776:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303972905 00000100:00000001:2.0:1788878361.081794:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878361.081801:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878361.081808:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.081819:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788878361.081828:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878361.081835:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878361.081839:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878361/real 1788878361] req@ffff94e50449d180 x1875765290055552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878361.081841:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878361.081850:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878361.081866:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878361.081872:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878361.081888:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:2.0:1788878361.081891:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788878361.081893:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878361.081943:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9299 00000100:00080000:3.0:1788878361.081950:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878361.081952:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878361.081955:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878361.081960:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878361.081963:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878361.081966:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878361.081970:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878361.081972:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878361.081975:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878361.081980:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878361.081984:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878361.081987:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878361.081992:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878361.081994:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878361.081996:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878361.082001:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000400:00000001:2.0:1788878361.082005:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00000001:3.0:1788878361.082006:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878361.082013:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878361.082020:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878361.082022:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:3.0:1788878361.082024:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082030:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878361.082031:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878361.082038:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878361.082039:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878361.082046:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082048:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878361.082054:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878361.082057:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878361.082068:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290055552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878361.082080:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290055936/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878361.082111:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878361.082118:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878361.082123:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878361.082128:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082135:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878361.082140:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082144:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878361.082151:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082184:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878361.082189:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878361.082197:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878361.082215:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878361.082219:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878361.082225:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290055552:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878361.082232:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082235:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878361.082237:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878361.082240:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878361.082246:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878361.082249:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878361.082252:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878361.082256:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878361.082258:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878361.082262:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878361.082264:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082267:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082275:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878361.082277:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082280:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878361.082282:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878361.082285:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878361.082287:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878361.082289:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878361.082292:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878361.082295:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878361.082298:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878361.082302:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290055680:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878361.082306:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878361.082309:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878361.082311:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878361.082314:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878361.082318:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082327:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878361.082329:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:2.0:1788878361.082335:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878361.082356:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878361.082363:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878361.082366:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878361.082368:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082370:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878361.082374:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082379:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303972364 00000100:00000001:2.0:1788878361.082383:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303972905 00000100:00000001:2.0:1788878361.082386:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303972364 00000100:00000001:2.0:1788878361.082388:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303972905 00000100:00000001:2.0:1788878361.082392:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878361.082394:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082397:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878361.082402:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878361/real 1788878361] req@ffff94e50449c000 x1875765290055680/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878361.082423:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878361.082425:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878361.082428:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878361.082430:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878361.082432:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082434:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878361.082436:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878361.082441:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290055680/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878361.082449:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082452:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878361.082455:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082457:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878361.082458:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082460:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878361.082475:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878361.082478:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878361.082481:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878361.082484:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878361.082488:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290055680:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878361.082492:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878361.082494:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878361.082496:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878361.082499:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878361.082501:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878361.082504:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878361.082508:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290055808:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878361.082512:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878361.082514:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878361.082516:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878361.082519:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878361.082522:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082527:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878361.082529:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:2.0:1788878361.082533:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878361.082545:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878361.082549:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878361.082552:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878361.082554:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082556:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878361.082559:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082562:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303972362 00000100:00000001:2.0:1788878361.082565:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303972905 00000100:00000001:2.0:1788878361.082568:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303972362 00000100:00000001:2.0:1788878361.082571:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303972905 00000100:00000001:2.0:1788878361.082574:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878361.082576:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082578:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878361.082582:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878361/real 1788878361] req@ffff94e50449f480 x1875765290055808/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878361.082592:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878361.082593:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878361.082596:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878361.082597:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878361.082599:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082601:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878361.082603:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878361.082607:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290055808/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878361.082625:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082627:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878361.082630:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082632:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878361.082634:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082636:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878361.082648:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878361.082651:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878361.082654:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878361.082657:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878361.082660:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290055808:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878361.082665:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878361.082667:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878361.082669:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878361.082671:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878361.082673:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878361.082676:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878361.082679:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290055936:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878361.082683:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878361.082688:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878361.082689:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878361.082691:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878361.082694:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082699:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878361.082701:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:2.0:1788878361.082704:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878361.082715:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878361.082720:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878361.082722:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878361.082724:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082726:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878361.082729:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082732:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303972362 00000100:00000001:2.0:1788878361.082735:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303972905 00000100:00000001:2.0:1788878361.082738:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303972362 00000100:00000001:2.0:1788878361.082741:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303972905 00000100:00000001:2.0:1788878361.082744:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878361.082746:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082748:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878361.082753:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878361/real 1788878361] req@ffff94e50449c700 x1875765290055936/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878361.082762:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878361.082763:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878361.082766:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878361.082767:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878361.082769:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082771:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878361.082772:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878361.082777:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290055936/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878361.082784:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082787:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878361.082790:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082792:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878361.082794:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082796:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878361.082808:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878361.082811:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878361.082815:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878361.082817:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878361.082822:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290055936:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878361.082827:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082829:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878361.082831:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878361.082833:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878361.082835:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878361.082837:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878361.082838:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878361.082841:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878361.082843:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878361.082845:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878361.082847:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082863:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878361.082865:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878361.082867:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878361.082869:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878361.082871:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878361.082883:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878361.082889:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878361.082891:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878361.082893:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878361.082895:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082897:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878361.082899:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878361.082900:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878361.082932:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878361.082934:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878361.082935:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878361.082937:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878361.082939:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878361.082941:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878361.082943:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082946:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878361.082949:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878361.082951:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878361.082958:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878361.082960:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878366.201169:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878366.201182:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878366.201187:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878366.201193:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878366.201199:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878366.201209:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9305 00000100:00080000:2.0:1788878366.201223:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878366.201228:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878366.201231:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878366.201241:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878366.201245:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878366.201250:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878366.201254:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878366.201257:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878366.201259:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878366.201265:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878366.201268:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878366.201272:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878366.201276:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878366.201280:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.201282:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878366.201285:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878366.201300:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878366.201302:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.201306:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878366.201309:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878366.201315:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.201329:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878366.201342:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290056064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878366.201355:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878366.201362:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.201387:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878366.201392:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878366.201400:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878366.201402:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878366.201404:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878366.201408:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878366.201412:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9305 00000100:00080000:2.0:1788878366.201418:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878366.201422:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878366.201426:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878366.201433:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878366.201437:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878366.201441:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878366.201446:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878366.201450:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878366.201454:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878366.201467:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878366.201478:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878366.201486:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878366.201501:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878366.201510:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.201518:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878366.201527:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878366.201541:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.201547:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878366.201555:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878366.201559:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.201562:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878366.201564:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878366.201567:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.201575:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878366.201588:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290056192/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878366.201626:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878366.201631:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.201637:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878366.201641:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878366.201646:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878366.201654:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878366.201659:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878366.201666:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878366.201672:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9305 00000100:00080000:2.0:1788878366.201679:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878366.201681:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878366.201684:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878366.201689:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878366.201691:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878366.201694:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878366.201698:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878366.201701:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878366.201703:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878366.201707:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878366.201710:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878366.201713:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878366.201720:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878366.201728:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.201731:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878366.201735:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878366.201739:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.201744:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878366.201754:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878366.201761:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.201775:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878366.201783:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878366.201786:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.201793:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878366.201806:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290056320/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878366.201819:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878366.201825:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.201831:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878366.201840:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878366.201846:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878366.201853:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878366.201857:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878366.201868:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878366.201878:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9305 00000100:00080000:2.0:1788878366.201887:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878366.201892:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878366.201895:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878366.201900:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878366.201951:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878366.201954:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878366.201958:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878366.201967:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878366.201972:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878366.201989:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878366.202003:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878366.202013:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878366.202026:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878366.202041:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.202050:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878366.202057:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878366.202067:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.202074:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878366.202087:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878366.202091:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.202098:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878366.202101:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878366.202108:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.202117:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878366.202130:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290056448/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878366.202163:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878366.202172:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878366.202182:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878371.321093:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878371.321103:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878371.321107:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878371.321135:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878371.321145:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878371.321156:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9310 00000100:00080000:3.0:1788878371.321181:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878371.321184:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878371.321187:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878371.321196:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878371.321200:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878371.321204:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878371.321207:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878371.321211:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878371.321214:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878371.321223:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878371.321229:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878371.321234:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878371.321240:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878371.321246:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878371.321251:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878371.321255:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878371.321273:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878371.321296:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878371.321304:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878371.321317:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878371.321327:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878371.321347:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878371.321366:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290056576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878371.321387:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878371.321399:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878371.321431:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878371.321440:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878371.321445:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878371.321449:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878371.321453:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878371.321460:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878371.321467:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9310 00000100:00080000:3.0:1788878371.321477:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878371.321480:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878371.321482:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878371.321489:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878371.321492:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788878371.321497:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788878371.321498:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788878371.321504:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788878371.321507:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1788878371.321511:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878371.321513:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878371.321520:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:2.0:1788878371.321525:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878371.321527:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1788878371.321534:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878371.321535:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788878371.321548:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878371.321553:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878371.321562:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878371.321564:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290056576:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788878371.321567:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878371.321579:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878371.321582:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878371.321589:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878371.321591:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878371.321595:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:2.0:1788878371.321598:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788878371.321603:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788878371.321604:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1788878371.321612:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878371.321614:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878371.321617:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878371.321631:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878371.321636:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878371.321641:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878371.321643:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788878371.321645:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878371.321653:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 02000000:00000001:3.0:1788878371.321655:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.321664:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878371.321671:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878371.321684:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290056704/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878371.321695:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878371.321699:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878371.321706:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878371.321709:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878371.321713:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878371.321715:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878371.321717:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878371.321721:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878371.321724:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9310 00000100:00080000:3.0:1788878371.321729:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878371.321733:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000800:00000100:2.0:1788878371.321738:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878371.321739:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878371.321757:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878371.321763:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878371.321766:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.321769:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878371.321775:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.321781:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303983145 00000100:00000001:2.0:1788878371.321792:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878371.321801:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303983145 00000100:00000001:2.0:1788878371.321805:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878371.321809:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878371.321812:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.321816:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878371.321824:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878371/real 1788878371] req@ffff94e50449fb80 x1875765290056576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878371.321839:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878371.321841:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878371.321847:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878371.321849:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878371.321851:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.321855:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878371.321857:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878371.321863:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290056576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878371.321873:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.321877:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878371.321880:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.321882:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878371.321884:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.321887:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878371.321889:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878371.321893:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878371.321898:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878371.321931:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878371.321939:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290056576:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878371.321947:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.321951:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878371.321954:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878371.321957:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878371.321963:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878371.321967:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878371.321970:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878371.321975:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878371.321978:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878371.321983:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878371.321986:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.321989:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.322010:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878371.322013:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.322017:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878371.322019:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878371.322021:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878371.322024:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878371.322026:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878371.322028:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878371.322031:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878371.322034:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878371.322038:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290056704:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878371.322042:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878371.322044:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878371.322047:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878371.322049:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878371.322053:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.322062:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878371.322064:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:2.0:1788878371.322070:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878371.322090:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878371.322098:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878371.322101:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878371.322108:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.322139:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878371.322145:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.322151:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303982604 00000100:00000001:2.0:1788878371.322167:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303983145 00000100:00000001:2.0:1788878371.322171:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303982604 00000100:00000001:2.0:1788878371.322175:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303983145 00000100:00000001:2.0:1788878371.322178:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878371.322181:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.322184:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878371.322190:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878371/real 1788878371] req@ffff94e50449d880 x1875765290056704/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878371.322201:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878371.322203:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878371.322207:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878371.322213:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878371.322220:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788878371.322222:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788878371.322225:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878371.322232:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788878371.322233:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:2.0:1788878371.322242:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290056704/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788878371.322244:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878371.322260:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878371.322265:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878371.322273:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878371.322275:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878371.322281:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.322285:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878371.322289:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.322292:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878371.322310:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878371.322313:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878371.322318:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878371.322325:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878371.322331:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290056704:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878371.322339:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.322343:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878371.322348:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878371.322355:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878371.322362:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878371.322366:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878371.322371:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878371.322376:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878371.322378:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878371.322386:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878371.322394:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878371.322401:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.322410:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788878371.322411:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878371.322419:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878371.322425:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.322426:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.322439:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788878371.322439:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878371.322446:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878371.322451:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878371.322458:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878371.322461:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878371.322466:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878371.322470:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878371.322473:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878371.322481:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878371.322482:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878371.322485:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878371.322487:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878371.322490:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878371.322498:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878371.322506:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290056832/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878371.322516:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878371.322520:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878371.322537:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878371.322542:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878371.322547:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878371.322549:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878371.322551:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878371.322555:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878371.322559:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9310 00000100:00080000:3.0:1788878371.322567:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878371.322572:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878371.322574:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878371.322580:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788878371.322581:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878371.322588:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788878371.322589:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788878371.322592:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878371.322595:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788878371.322600:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878371.322602:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878371.322608:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878371.322610:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878371.322616:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788878371.322618:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878371.322625:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788878371.322626:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:2.0:1788878371.322633:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878371.322635:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878371.322641:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1788878371.322646:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290056832:192.168.203.129@tcp:8:kworker.0 02000000:00000001:3.0:1788878371.322649:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878371.322660:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.322662:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788878371.322665:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:2.0:1788878371.322668:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:3.0:1788878371.322671:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788878371.322674:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788878371.322677:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878371.322680:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878371.322683:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878371.322689:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878371.322692:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878371.322697:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878371.322699:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878371.322702:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878371.322706:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.322707:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878371.322712:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:3.0:1788878371.322716:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.322719:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1788878371.322730:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290056960/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878371.322741:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878371.322745:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788878371.322753:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878371.322755:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.322767:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878371.322775:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878371.322777:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.322780:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878371.322784:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.322790:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303982602 00000100:00000001:2.0:1788878371.322797:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303983145 00000100:00000001:2.0:1788878371.322800:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303982602 00000100:00000001:2.0:1788878371.322803:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303983145 00000100:00000001:2.0:1788878371.322806:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878371.322809:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.322811:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878371.322817:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878371/real 1788878371] req@ffff94e50449c380 x1875765290056832/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878371.322827:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878371.322828:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878371.322832:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878371.322834:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878371.322836:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.322838:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878371.322840:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878371.322844:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290056832/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878371.322853:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.322856:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878371.322859:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.322861:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878371.322863:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.322865:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878371.322878:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878371.322881:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878371.322884:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878371.322887:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878371.322891:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290056832:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878371.322896:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.322899:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878371.322944:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878371.322948:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878371.322954:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878371.322957:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878371.322959:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878371.322964:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878371.322968:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878371.322972:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878371.322974:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.322978:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.322983:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878371.322985:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.322988:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878371.322999:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878371.323001:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878371.323004:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878371.323007:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878371.323010:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878371.323013:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878371.323016:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878371.323020:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290056960:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878371.323024:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878371.323026:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878371.323029:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878371.323031:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878371.323035:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.323041:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878371.323043:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:2.0:1788878371.323047:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878371.323062:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878371.323067:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878371.323070:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878371.323072:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.323074:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878371.323077:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.323081:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303982603 00000100:00000001:2.0:1788878371.323084:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303983145 00000100:00000001:2.0:1788878371.323088:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303982603 00000100:00000001:2.0:1788878371.323091:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303983145 00000100:00000001:2.0:1788878371.323094:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878371.323096:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.323099:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878371.323103:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878371/real 1788878371] req@ffff94e50449ce00 x1875765290056960/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878371.323140:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878371.323142:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878371.323146:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878371.323148:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878371.323150:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.323152:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878371.323154:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878371.323159:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290056960/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878371.323168:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.323171:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878371.323174:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.323176:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878371.323177:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.323179:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878371.323191:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878371.323194:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878371.323198:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878371.323200:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878371.323212:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290056960:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878371.323217:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.323219:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878371.323221:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878371.323223:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878371.323226:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878371.323227:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878371.323229:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878371.323232:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878371.323233:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878371.323236:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878371.323237:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.323240:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878371.323243:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878371.323245:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878371.323247:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878371.323249:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878376.441119:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878376.441135:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878376.441138:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878376.441144:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878376.441153:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878376.441166:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9315 00000100:00080000:3.0:1788878376.441185:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878376.441191:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878376.441195:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878376.441210:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878376.441215:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878376.441223:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878376.441229:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878376.441233:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878376.441236:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878376.441245:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878376.441250:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878376.441256:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878376.441261:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878376.441265:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878376.441268:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878376.441272:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878376.441289:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878376.441292:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878376.441296:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878376.441301:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878376.441308:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878376.441325:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878376.441338:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290057088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878376.441353:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878376.441377:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878376.441403:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878376.441408:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878376.441412:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878376.441414:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878376.441416:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878376.441420:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878376.441424:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9315 00000100:00080000:3.0:1788878376.441429:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878376.441432:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878376.441435:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878376.441441:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878376.441445:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878376.441448:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878376.441452:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878376.441455:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878376.441457:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878376.441462:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878376.441465:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878376.441466:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878376.441472:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878376.441478:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788878376.441483:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878376.441484:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878376.441492:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788878376.441493:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.441497:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878376.441499:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878376.441508:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788878376.441510:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1788878376.441517:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878376.441519:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878376.441523:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878376.441525:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878376.441532:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878376.441538:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878376.441539:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290057088:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878376.441542:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878376.441546:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878376.441552:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.441554:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788878376.441562:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878376.441563:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878376.441571:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878376.441577:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1788878376.441580:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290057216/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788878376.441586:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878376.441599:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878376.441603:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878376.441609:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878376.441613:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878376.441617:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878376.441619:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788878376.441620:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788878376.441623:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878376.441626:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000100:00000001:3.0:1788878376.441631:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788878376.441636:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788878376.441637:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9315 00000100:00080000:3.0:1788878376.441645:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878376.441649:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878376.441651:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878376.441677:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878376.441679:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878376.441699:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878376.441702:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878376.441704:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878376.441706:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878376.441710:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878376.441713:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878376.441716:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878376.441721:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:2.0:1788878376.441724:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788878376.441725:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878376.441733:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878376.441737:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878376.441740:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878376.441743:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878376.441751:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878376.441753:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.441755:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878376.441756:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878376.441762:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878376.441766:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.441769:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878376.441776:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.441777:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.441786:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:3.0:1788878376.441789:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290057344/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878376.441799:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878376.441801:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878376.441808:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.441831:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303988265 00000100:00000001:3.0:1788878376.441831:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878376.441843:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878376.441847:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878376.441849:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878376.441851:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878376.441856:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878376.441857:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878376.441866:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9315 00000100:00000001:2.0:1788878376.441874:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303988265 00000100:00080000:3.0:1788878376.441874:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878376.441884:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878376.441890:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878376.441891:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878376.441900:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788878376.441904:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788878376.441906:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878376.441912:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788878376.441914:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1788878376.441922:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:2.0:1788878376.441931:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878376/real 1788878376] req@ffff94e50449e680 x1875765290057088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788878376.441978:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878376.441984:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878376.441987:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878376.441999:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878376.442008:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878376.442017:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878376.442024:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:2.0:1788878376.442026:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1788878376.442028:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788878376.442032:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878376.442035:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878376.442041:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878376.442043:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000008:00000001:3.0:1788878376.442045:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442068:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878376.442069:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442074:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442081:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788878376.442081:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878376.442106:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442109:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878376.442113:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878376.442118:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878376.442123:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878376.442124:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290057088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878376.442134:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442143:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442151:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:3.0:1788878376.442154:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290057472/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878376.442156:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442162:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878376.442166:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442174:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878376.442179:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878376.442189:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788878376.442193:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00080000:2.0:1788878376.442199:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878376.442205:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442206:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788878376.442219:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878376.442220:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290057088:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878376.442234:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442238:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878376.442241:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878376.442244:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878376.442250:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878376.442253:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878376.442256:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878376.442261:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878376.442264:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878376.442268:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878376.442270:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442273:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442283:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878376.442286:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442289:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878376.442291:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878376.442293:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878376.442295:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878376.442297:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878376.442300:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878376.442302:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878376.442305:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878376.442309:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290057216:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878376.442312:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878376.442315:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878376.442317:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878376.442319:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878376.442323:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442331:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878376.442333:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:2.0:1788878376.442339:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878376.442358:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878376.442365:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878376.442369:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878376.442372:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442375:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878376.442381:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442406:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303987724 00000100:00000001:2.0:1788878376.442412:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303988265 00000100:00000001:2.0:1788878376.442417:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303987724 00000100:00000001:2.0:1788878376.442421:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303988265 00000100:00000001:2.0:1788878376.442426:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878376.442430:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442435:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878376.442444:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878376/real 1788878376] req@ffff94e50449df80 x1875765290057216/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878376.442463:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878376.442466:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878376.442472:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878376.442475:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878376.442478:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442482:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878376.442484:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878376.442491:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290057216/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878376.442501:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442504:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878376.442508:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442510:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878376.442511:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442514:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878376.442533:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878376.442536:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878376.442540:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878376.442543:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878376.442547:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290057216:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878376.442552:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878376.442554:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878376.442556:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878376.442559:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878376.442562:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878376.442565:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878376.442569:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290057344:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878376.442573:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878376.442576:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878376.442578:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878376.442581:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878376.442585:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442594:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878376.442595:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:2.0:1788878376.442601:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878376.442619:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878376.442625:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878376.442628:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878376.442630:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442632:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878376.442635:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442640:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303987722 00000100:00000001:2.0:1788878376.442643:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303988265 00000100:00000001:2.0:1788878376.442646:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303987722 00000100:00000001:2.0:1788878376.442649:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303988265 00000100:00000001:2.0:1788878376.442652:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878376.442654:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442656:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878376.442661:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878376/real 1788878376] req@ffff94e50449ce00 x1875765290057344/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878376.442671:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878376.442689:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878376.442694:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878376.442696:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878376.442699:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442702:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878376.442705:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878376.442712:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290057344/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878376.442736:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442741:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878376.442746:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442749:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878376.442752:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442755:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878376.442778:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878376.442783:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878376.442787:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878376.442790:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878376.442794:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290057344:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878376.442816:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878376.442818:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878376.442820:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878376.442823:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878376.442826:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878376.442828:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878376.442832:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290057472:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878376.442836:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878376.442839:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878376.442841:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878376.442843:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878376.442847:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442855:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878376.442857:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:2.0:1788878376.442862:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878376.442880:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878376.442887:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878376.442889:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878376.442891:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442894:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878376.442897:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442902:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303987722 00000100:00000001:2.0:1788878376.442905:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303988265 00000100:00000001:2.0:1788878376.442908:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303987722 00000100:00000001:2.0:1788878376.442911:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303988265 00000100:00000001:2.0:1788878376.442913:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878376.442916:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.442918:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878376.442923:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878376/real 1788878376] req@ffff94e50449c380 x1875765290057472/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878376.442932:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878376.442933:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878376.442936:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878376.442938:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878376.442940:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442942:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878376.442944:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878376.442978:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290057472/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878376.442988:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442991:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878376.442994:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.442996:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878376.442997:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.443000:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878376.443017:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878376.443019:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878376.443023:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878376.443026:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878376.443031:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290057472:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878376.443036:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.443038:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878376.443057:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878376.443060:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878376.443063:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878376.443064:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878376.443066:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878376.443069:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878376.443070:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878376.443073:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878376.443075:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.443077:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878376.443078:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878376.443080:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878376.443082:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878376.443083:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878376.443085:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878376.443108:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878376.443111:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878376.443114:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878376.443116:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.443119:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878376.443122:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878376.443123:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878376.443126:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878376.443128:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878376.443130:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878376.443132:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878376.443135:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878376.443137:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878376.443139:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.443142:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878376.443157:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878376.443159:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878376.443163:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878376.443165:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878381.561016:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878381.561027:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878381.561029:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878381.561033:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878381.561038:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878381.561045:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9320 00000100:00080000:3.0:1788878381.561091:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878381.561095:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878381.561097:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878381.561105:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878381.561108:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878381.561112:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878381.561132:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878381.561134:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878381.561136:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878381.561140:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878381.561142:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878381.561145:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878381.561149:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878381.561151:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878381.561154:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878381.561156:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878381.561173:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878381.561174:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878381.561177:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878381.561180:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878381.561185:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878381.561196:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878381.561205:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290057600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878381.561215:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878381.561220:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878381.561234:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878381.561237:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878381.561241:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878381.561242:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878381.561244:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878381.561247:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878381.561249:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9320 00000100:00080000:3.0:1788878381.561253:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878381.561255:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878381.561258:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878381.561261:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878381.561263:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878381.561265:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878381.561267:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878381.561269:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878381.561270:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878381.561275:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878381.561278:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788878381.561281:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878381.561283:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878381.561288:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.561289:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878381.561294:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878381.561294:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878381.561301:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878381.561303:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878381.561305:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878381.561308:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878381.561310:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878381.561312:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878381.561317:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878381.561319:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290057600:192.168.203.129@tcp:250:kworker.0 00000008:00000001:3.0:1788878381.561324:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.561329:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788878381.561330:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878381.561335:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878381.561336:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878381.561339:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878381.561340:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878381.561345:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878381.561346:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878381.561350:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878381.561353:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878381.561353:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878381.561364:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878381.561372:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290057728/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878381.561374:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878381.561378:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:2.0:1788878381.561386:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878381.561394:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878381.561397:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878381.561401:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878381.561403:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878381.561406:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878381.561408:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878381.561409:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878381.561412:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878381.561415:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9320 00000100:00080000:3.0:1788878381.561418:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878381.561420:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878381.561421:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878381.561424:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878381.561427:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:2.0:1788878381.561428:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:3.0:1788878381.561430:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878381.561435:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878381.561437:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878381.561439:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878381.561443:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:3.0:1788878381.561445:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878381.561450:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.561452:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788878381.561453:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878381.561457:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878381.561459:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878381.561461:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788878381.561466:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878381.561469:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878381.561470:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788878381.561472:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878381.561476:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303993385 00000008:00000001:3.0:1788878381.561477:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878381.561483:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878381.561489:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878381.561491:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788878381.561492:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878381.561498:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878381.561501:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878381.561503:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.561509:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303993385 00000100:00000001:3.0:1788878381.561511:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.561517:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878381.561525:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788878381.561525:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290057856/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878381.561531:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.561536:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878381.561545:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878381/real 1788878381] req@ffff94e50449d880 x1875765290057600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878381.561548:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878381.561556:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788878381.561566:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788878381.561568:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788878381.561570:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1788878381.561574:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788878381.561577:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:3.0:1788878381.561581:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878381.561582:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878381.561586:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788878381.561587:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788878381.561591:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878381.561592:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878381.561597:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878381.561599:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00100000:2.0:1788878381.561604:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290057600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878381.561607:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9320 00000100:00080000:3.0:1788878381.561617:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878381.561619:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878381.561621:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878381.561625:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878381.561626:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878381.561635:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788878381.561636:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788878381.561638:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1788878381.561641:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788878381.561642:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788878381.561645:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878381.561646:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878381.561651:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1788878381.561652:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00080000:2.0:1788878381.561657:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788878381.561658:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878381.561663:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878381.561666:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878381.561673:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:3.0:1788878381.561673:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878381.561680:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878381.561682:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290057600:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788878381.561684:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878381.561689:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878381.561690:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878381.561692:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878381.561696:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878381.561697:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878381.561700:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788878381.561702:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:2.0:1788878381.561705:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000008:00100000:3.0:1788878381.561706:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000020:00000001:2.0:1788878381.561709:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000008:00000001:3.0:1788878381.561711:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878381.561713:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788878381.561717:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878381.561720:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878381.561722:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:3.0:1788878381.561724:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878381.561725:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 02000000:00000001:3.0:1788878381.561728:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.561729:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878381.561733:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878381.561734:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878381.561737:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878381.561741:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.561743:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878381.561746:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.561747:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.561756:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878381.561759:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878381.561761:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00100000:3.0:1788878381.561762:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290057984/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878381.561764:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878381.561767:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878381.561773:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878381.561777:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878381.561779:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788878381.561782:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878381.561784:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878381.561790:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290057728:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878381.561793:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.561795:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878381.561798:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878381.561800:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878381.561802:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878381.561806:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.561811:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878381.561814:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:2.0:1788878381.561818:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878381.561831:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878381.561835:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878381.561837:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878381.561839:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.561841:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878381.561843:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.561847:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303992843 00000100:00000001:2.0:1788878381.561850:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303993385 00000100:00000001:2.0:1788878381.561853:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303992843 00000100:00000001:2.0:1788878381.561855:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303993385 00000100:00000001:2.0:1788878381.561857:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878381.561858:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.561860:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878381.561864:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878381/real 1788878381] req@ffff94e50449fb80 x1875765290057728/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878381.561871:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878381.561872:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878381.561874:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878381.561875:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878381.561877:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.561878:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878381.561880:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878381.561883:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290057728/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878381.561890:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.561892:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878381.561894:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.561895:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878381.561896:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.561898:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878381.561944:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878381.561950:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878381.561955:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878381.561958:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878381.561963:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290057728:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878381.561969:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878381.561971:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878381.561973:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878381.561976:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878381.561979:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878381.561982:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878381.561986:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290057856:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878381.561989:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878381.561991:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878381.561993:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878381.561994:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878381.561998:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.562004:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878381.562005:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:2.0:1788878381.562009:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878381.562022:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878381.562027:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878381.562029:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878381.562031:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.562033:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878381.562035:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.562038:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303992842 00000100:00000001:2.0:1788878381.562040:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303993385 00000100:00000001:2.0:1788878381.562043:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303992842 00000100:00000001:2.0:1788878381.562045:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303993385 00000100:00000001:2.0:1788878381.562048:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878381.562051:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.562054:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878381.562059:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878381/real 1788878381] req@ffff94e50449c700 x1875765290057856/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878381.562090:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878381.562092:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878381.562096:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878381.562099:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878381.562100:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.562104:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878381.562106:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878381.562112:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290057856/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878381.562123:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.562126:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878381.562129:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.562132:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878381.562134:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.562136:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878381.562148:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878381.562150:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878381.562153:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878381.562155:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878381.562159:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290057856:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878381.562171:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.562173:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878381.562174:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878381.562176:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878381.562179:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878381.562180:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878381.562181:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878381.562184:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878381.562185:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878381.562187:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878381.562188:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.562190:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878381.562191:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878381.562192:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878381.562193:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878381.562194:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878381.562195:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878381.562197:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878381.562197:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878381.562199:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878381.562200:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.562201:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.562205:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878381.562206:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.562208:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878381.562209:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878381.562210:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878381.562212:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878381.562213:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878381.562215:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878381.562216:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878381.562218:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878381.562221:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290057984:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878381.562224:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878381.562226:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878381.562228:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878381.562229:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878381.562233:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.562238:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878381.562239:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:2.0:1788878381.562243:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878381.562256:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878381.562260:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878381.562262:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878381.562263:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.562265:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878381.562267:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.562270:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303992842 00000100:00000001:2.0:1788878381.562272:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303993385 00000100:00000001:2.0:1788878381.562274:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303992842 00000100:00000001:2.0:1788878381.562276:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303993385 00000100:00000001:2.0:1788878381.562278:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878381.562280:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.562282:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878381.562285:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878381/real 1788878381] req@ffff94e50449f480 x1875765290057984/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878381.562292:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878381.562293:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878381.562295:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878381.562296:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878381.562298:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.562299:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878381.562301:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878381.562304:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290057984/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878381.562310:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.562312:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878381.562314:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.562315:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878381.562317:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.562319:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878381.562330:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878381.562332:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878381.562335:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878381.562337:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878381.562340:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290057984:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878381.562343:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.562345:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878381.562346:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878381.562347:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878381.562348:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878381.562349:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878381.562350:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878381.562352:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878381.562370:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878381.562372:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878381.562374:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.562376:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878381.562378:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878381.562380:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878381.562382:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878381.562383:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878386.681096:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878386.681114:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878386.681118:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878386.681124:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878386.681131:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878386.681144:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9325 00000100:00080000:3.0:1788878386.681162:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878386.681166:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878386.681169:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878386.681181:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878386.681186:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878386.681191:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878386.681195:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878386.681197:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878386.681200:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878386.681206:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878386.681209:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878386.681214:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878386.681222:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878386.681226:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878386.681229:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878386.681233:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878386.681253:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878386.681255:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878386.681260:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878386.681265:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878386.681273:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878386.681294:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878386.681314:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290058112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878386.681335:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878386.681346:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878386.681389:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878386.681396:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878386.681403:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878386.681406:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878386.681411:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878386.681416:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878386.681420:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9325 00000100:00080000:3.0:1788878386.681427:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878386.681430:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878386.681432:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878386.681437:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878386.681439:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878386.681442:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878386.681445:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878386.681448:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878386.681450:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878386.681455:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878386.681458:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878386.681462:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878386.681471:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878386.681474:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878386.681477:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878386.681484:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878386.681489:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:3.0:1788878386.681489:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878386.681503:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.681507:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878386.681512:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788878386.681513:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878386.681521:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788878386.681521:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.681533:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878386.681534:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878386.681539:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878386.681547:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878386.681549:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878386.681570:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878386.681574:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878386.681581:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878386.681586:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290058240/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788878386.681599:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290058112:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878386.681625:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878386.681635:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878386.681641:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878386.681641:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788878386.681654:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878386.681658:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878386.681684:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878386.681686:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878386.681694:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878386.681703:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878386.681708:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878386.681711:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878386.681720:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878386.681721:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878386.681734:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9325 00000100:00000001:2.0:1788878386.681751:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00080000:3.0:1788878386.681757:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878386.681766:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878386.681768:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878386.681774:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878386.681785:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878386.681788:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878386.681794:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878386.681798:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878386.681812:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878386.681815:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878386.681821:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878386.681825:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878386.681830:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878386.681836:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878386.681839:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878386.681841:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878386.681847:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878386.681852:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788878386.681855:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878386.681858:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878386.681867:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878386.681870:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878386.681875:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878386.681879:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788878386.681879:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878386.681894:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.681898:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878386.681903:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.681906:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878386.681917:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.681925:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4303998506 00000100:00000001:2.0:1788878386.681940:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878386.681951:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4303998506 00000100:00000001:2.0:1788878386.681957:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878386.681960:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.681964:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788878386.681973:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290058368/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878386.681986:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878386.681991:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878386.681995:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878386.682001:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878386.682007:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878386.682010:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878386.682013:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878386.682017:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878386.682021:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9325 00000100:00000001:2.0:1788878386.682025:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878386.682031:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878386.682033:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788878386.682068:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878386.682074:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:2.0:1788878386.682085:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878386/real 1788878386] req@ffff94e50449c000 x1875765290058112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788878386.682088:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878386.682095:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878386.682104:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878386.682121:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878386.682131:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000400:00000001:2.0:1788878386.682132:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788878386.682136:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:2.0:1788878386.682137:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878386.682155:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:3.0:1788878386.682156:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878386.682167:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878386.682167:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.682181:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878386.682183:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878386.682198:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788878386.682200:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878386.682204:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878386.682214:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878386.682224:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:2.0:1788878386.682226:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290058112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788878386.682230:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878386.682251:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878386.682269:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.682275:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878386.682280:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878386.682283:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878386.682286:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.682294:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878386.682299:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878386.682300:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878386.682307:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878386.682311:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878386.682313:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878386.682319:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878386.682326:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878386.682328:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788878386.682350:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878386.682372:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878386.682380:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878386.682383:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290058496/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788878386.682392:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290058112:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878386.682421:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.682436:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878386.682439:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878386.682441:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1788878386.682446:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878386.682448:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788878386.682464:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878386.682466:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878386.682479:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878386.682482:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878386.682492:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878386.682494:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878386.682497:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878386.682499:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.682503:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.682513:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878386.682516:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.682519:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878386.682522:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878386.682524:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878386.682527:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878386.682528:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878386.682531:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878386.682534:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878386.682538:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878386.682542:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290058240:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878386.682547:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878386.682550:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878386.682552:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878386.682554:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878386.682558:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.682585:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878386.682587:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:2.0:1788878386.682593:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878386.682614:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878386.682620:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878386.682623:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878386.682625:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.682628:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878386.682631:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.682637:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303997964 00000100:00000001:2.0:1788878386.682641:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303998506 00000100:00000001:2.0:1788878386.682644:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303997964 00000100:00000001:2.0:1788878386.682647:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303998506 00000100:00000001:2.0:1788878386.682649:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878386.682652:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.682654:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878386.682659:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878386/real 1788878386] req@ffff94e50449d180 x1875765290058240/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878386.682692:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878386.682694:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878386.682698:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878386.682701:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878386.682703:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.682706:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878386.682709:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878386.682715:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290058240/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878386.682728:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.682748:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878386.682753:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.682755:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878386.682758:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.682761:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878386.682783:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878386.682788:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878386.682793:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878386.682796:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878386.682817:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290058240:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878386.682822:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878386.682825:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878386.682827:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878386.682830:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878386.682832:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878386.682834:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878386.682838:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290058368:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878386.682842:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878386.682845:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878386.682847:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878386.682849:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878386.682853:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.682861:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878386.682863:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:2.0:1788878386.682868:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878386.682883:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878386.682890:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878386.682892:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878386.682894:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.682896:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878386.682899:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.682904:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303997962 00000100:00000001:2.0:1788878386.682907:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303998506 00000100:00000001:2.0:1788878386.682909:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303997962 00000100:00000001:2.0:1788878386.682912:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303998506 00000100:00000001:2.0:1788878386.682917:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878386.682919:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.682921:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878386.682926:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878386/real 1788878386] req@ffff94e50449f800 x1875765290058368/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878386.682953:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878386.682955:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878386.682959:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878386.682961:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878386.682962:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.682965:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878386.682967:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878386.682971:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290058368/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878386.682981:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.682983:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878386.682986:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.682988:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878386.682990:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.682992:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878386.683004:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878386.683040:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878386.683071:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878386.683074:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878386.683078:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290058368:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878386.683102:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878386.683104:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878386.683106:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878386.683110:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878386.683113:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878386.683115:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878386.683119:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290058496:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878386.683124:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878386.683126:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878386.683128:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878386.683131:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878386.683135:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.683142:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878386.683144:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:2.0:1788878386.683149:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878386.683183:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878386.683190:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878386.683193:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878386.683195:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.683197:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878386.683200:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.683205:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4303997963 00000100:00000001:2.0:1788878386.683208:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4303998506 00000100:00000001:2.0:1788878386.683211:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4303997963 00000100:00000001:2.0:1788878386.683214:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4303998506 00000100:00000001:2.0:1788878386.683217:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878386.683219:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.683222:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878386.683227:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878386/real 1788878386] req@ffff94e50449ca80 x1875765290058496/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878386.683236:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878386.683237:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878386.683240:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878386.683242:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878386.683244:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.683246:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878386.683249:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878386.683253:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290058496/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878386.683270:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.683273:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878386.683276:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.683278:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878386.683279:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.683281:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878386.683293:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878386.683296:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878386.683300:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878386.683302:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878386.683306:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290058496:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878386.683310:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.683313:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878386.683314:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878386.683317:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878386.683320:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878386.683321:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878386.683323:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878386.683326:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878386.683328:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878386.683330:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878386.683332:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.683334:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878386.683336:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878386.683337:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878386.683356:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878386.683357:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878386.683359:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878386.683360:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878386.683361:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878386.683363:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878386.683364:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.683366:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878386.683367:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878386.683369:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878386.683371:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878386.683373:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878386.683374:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878386.683376:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878386.683377:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878386.683379:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878386.683380:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.683382:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878386.683386:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878386.683388:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878386.683390:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878386.683391:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878391.800996:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878391.801007:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878391.801010:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878391.801014:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878391.801038:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878391.801048:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9330 00000100:00080000:3.0:1788878391.801060:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878391.801064:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878391.801066:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878391.801074:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878391.801077:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878391.801081:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878391.801083:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878391.801085:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878391.801087:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878391.801092:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878391.801094:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878391.801097:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878391.801101:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878391.801104:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878391.801107:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878391.801109:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878391.801123:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878391.801125:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878391.801131:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878391.801134:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878391.801140:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878391.801155:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878391.801168:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290058624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878391.801185:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878391.801191:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878391.801212:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878391.801216:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878391.801221:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878391.801224:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878391.801226:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878391.801231:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878391.801235:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9330 00000100:00080000:3.0:1788878391.801242:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878391.801246:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878391.801249:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878391.801255:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878391.801258:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878391.801262:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878391.801266:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878391.801268:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878391.801270:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878391.801275:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878391.801276:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878391.801280:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878391.801284:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1788878391.801288:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878391.801289:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878391.801299:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878391.801300:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878391.801302:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878391.801307:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878391.801312:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878391.801315:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878391.801317:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878391.801323:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878391.801329:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878391.801330:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290058624:192.168.203.129@tcp:250:kworker.0 00000008:00000001:3.0:1788878391.801334:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878391.801340:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801347:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878391.801349:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878391.801353:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878391.801355:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878391.801356:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878391.801361:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878391.801362:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878391.801364:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878391.801373:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878391.801376:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878391.801382:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878391.801386:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290058752/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878391.801397:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878391.801399:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801402:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878391.801403:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801406:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00080000:3.0:1788878391.801409:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788878391.801412:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788878391.801417:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878391.801421:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878391.801423:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878391.801426:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878391.801429:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9330 00000100:00080000:3.0:1788878391.801445:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878391.801447:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878391.801448:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878391.801452:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878391.801453:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878391.801457:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000800:00000100:2.0:1788878391.801458:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:3.0:1788878391.801463:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878391.801468:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878391.801470:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878391.801476:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:3.0:1788878391.801476:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878391.801483:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801484:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788878391.801486:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878391.801489:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878391.801492:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878391.801493:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788878391.801499:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801502:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878391.801503:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878391.801508:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878391.801510:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304003625 00000008:00000001:3.0:1788878391.801512:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878391.801518:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801524:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788878391.801525:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878391.801528:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878391.801532:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878391.801534:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878391.801536:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878391.801541:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878391.801546:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290058880/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878391.801550:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304003625 00000100:00000001:3.0:1788878391.801559:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878391.801562:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878391.801563:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801572:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878391.801572:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878391.801581:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788878391.801582:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801589:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788878391.801590:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878391.801594:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878391.801596:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788878391.801599:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878391/real 1788878391] req@ffff94e50449e300 x1875765290058624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878391.801601:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878391.801608:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9330 00000100:00080000:3.0:1788878391.801617:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878391.801624:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000400:00000001:2.0:1788878391.801625:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788878391.801626:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:2.0:1788878391.801630:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1788878391.801632:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878391.801636:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788878391.801638:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000020:00000001:3.0:1788878391.801639:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878391.801645:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878391.801646:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878391.801652:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878391.801653:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878391.801657:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878391.801659:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878391.801664:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788878391.801666:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878391.801674:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878391.801675:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290058624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788878391.801679:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878391.801685:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878391.801689:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878391.801690:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801697:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878391.801697:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878391.801703:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878391.801705:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1788878391.801712:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801713:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878391.801733:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878391.801735:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801739:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878391.801743:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878391.801743:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878391.801748:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878391.801749:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788878391.801752:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878391.801757:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00080000:2.0:1788878391.801758:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 02000000:00000001:3.0:1788878391.801762:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801764:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788878391.801770:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878391.801771:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290058624:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878391.801781:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878391.801782:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290059008/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878391.801784:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878391.801788:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878391.801793:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878391.801798:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1788878391.801799:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000020:00000001:2.0:1788878391.801803:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:3.0:1788878391.801804:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878391.801807:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788878391.801809:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878391.801813:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878391.801816:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878391.801819:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878391.801821:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.801824:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.801831:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878391.801833:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801836:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878391.801838:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878391.801840:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878391.801842:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878391.801844:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878391.801846:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878391.801849:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878391.801852:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878391.801856:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290058752:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878391.801859:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878391.801861:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878391.801862:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878391.801864:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878391.801866:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801872:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878391.801874:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:2.0:1788878391.801878:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878391.801891:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878391.801895:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878391.801897:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878391.801898:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801900:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878391.801902:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801906:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304003083 00000100:00000001:2.0:1788878391.801909:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304003625 00000100:00000001:2.0:1788878391.801911:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304003083 00000100:00000001:2.0:1788878391.801913:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304003625 00000100:00000001:2.0:1788878391.801916:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878391.801951:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.801953:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878391.801957:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878391/real 1788878391] req@ffff94e50449f100 x1875765290058752/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878391.801965:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878391.801967:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878391.801970:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878391.801971:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878391.801973:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.801974:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878391.801976:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878391.801980:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290058752/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878391.801990:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.801993:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878391.801997:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.801999:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878391.802001:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.802003:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878391.802037:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878391.802042:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878391.802048:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878391.802051:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878391.802056:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290058752:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878391.802062:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878391.802065:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878391.802067:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878391.802070:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878391.802073:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878391.802076:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878391.802079:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290058880:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878391.802082:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878391.802084:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878391.802086:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878391.802088:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878391.802091:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.802098:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878391.802099:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:2.0:1788878391.802103:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878391.802116:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878391.802121:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878391.802123:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878391.802124:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.802126:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878391.802128:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.802132:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304003082 00000100:00000001:2.0:1788878391.802134:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304003625 00000100:00000001:2.0:1788878391.802136:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304003082 00000100:00000001:2.0:1788878391.802138:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304003625 00000100:00000001:2.0:1788878391.802140:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878391.802141:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.802143:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878391.802147:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878391/real 1788878391] req@ffff94e50449ca80 x1875765290058880/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878391.802153:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878391.802154:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878391.802157:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878391.802158:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878391.802159:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.802161:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878391.802163:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878391.802166:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290058880/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878391.802178:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.802180:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878391.802182:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.802183:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878391.802185:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.802186:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878391.802196:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878391.802198:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878391.802201:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878391.802202:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878391.802205:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290058880:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878391.802208:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878391.802210:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878391.802211:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878391.802213:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878391.802215:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878391.802216:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878391.802219:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290059008:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878391.802221:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878391.802222:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878391.802224:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878391.802225:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878391.802228:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.802231:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878391.802233:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:2.0:1788878391.802236:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878391.802244:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878391.802247:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878391.802249:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878391.802250:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.802252:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878391.802254:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.802257:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304003082 00000100:00000001:2.0:1788878391.802259:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304003625 00000100:00000001:2.0:1788878391.802261:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304003082 00000100:00000001:2.0:1788878391.802263:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304003625 00000100:00000001:2.0:1788878391.802265:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878391.802266:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.802268:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878391.802271:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878391/real 1788878391] req@ffff94e50449f800 x1875765290059008/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878391.802277:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878391.802278:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878391.802280:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878391.802281:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878391.802282:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.802283:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878391.802284:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878391.802287:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290059008/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878391.802294:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.802297:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878391.802300:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.802303:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878391.802305:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.802307:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878391.802323:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878391.802327:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878391.802332:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878391.802335:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878391.802340:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290059008:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878391.802346:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.802349:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878391.802351:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878391.802353:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878391.802357:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878391.802359:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878391.802361:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878391.802364:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878391.802366:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878391.802369:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878391.802370:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.802372:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878391.802373:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878391.802374:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878391.802376:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878391.802377:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878391.802378:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878391.802380:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878391.802381:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878391.802382:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878391.802383:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.802384:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878391.802385:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878391.802386:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878391.802387:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878391.802388:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878391.802389:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878391.802390:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878391.802391:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878391.802392:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878391.802394:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.802395:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878391.802399:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878391.802400:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878391.802402:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878391.802403:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.921254:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878396.921260:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878396.921265:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878396.921270:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878396.921274:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878396.921282:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878396.921288:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878396.921293:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878396.921317:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290059136:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878396.921324:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878396.921331:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878396.921334:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878396.921341:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878396.921349:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.921419:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878396.921423:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 00000100:00000001:2.0:1788878396.921430:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878396.921473:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878396.921490:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878396.921499:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878396.921503:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.921510:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878396.921517:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.921524:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304008745 00000100:00000001:2.0:1788878396.921538:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878396.921549:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304008745 00000100:00000001:2.0:1788878396.921559:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878396.921567:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878396.921572:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.921576:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878396.921584:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878396/real 1788878396] req@ffff94e508978380 x1875765290059136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878396.921595:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878396.921598:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878396.921601:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878396.921604:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878396.921605:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.921608:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878396.921610:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878396.921614:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290059136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878396.921621:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.921624:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878396.921626:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.921628:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878396.921629:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.921630:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878396.921632:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878396.921635:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878396.921639:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878396.921641:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878396.921645:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290059136:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878396.921649:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.921650:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878396.921652:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878396.921654:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878396.921656:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878396.921658:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878396.921660:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878396.921663:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878396.921665:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878396.921667:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878396.921668:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.921670:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.921677:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878396.921678:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.921680:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878396.921682:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878396.921683:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878396.921685:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878396.921686:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878396.921688:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878396.921690:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878396.921692:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878396.921695:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290059264:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878396.921698:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878396.921699:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878396.921700:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878396.921702:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878396.921704:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.921709:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878396.921710:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000100:00000001:2.0:1788878396.921714:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878396.921724:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878396.921728:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878396.921730:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878396.921731:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.921733:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878396.921735:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.921750:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304008203 00000100:00000001:2.0:1788878396.921752:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304008745 00000100:00000001:2.0:1788878396.921755:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304008203 00000100:00000001:2.0:1788878396.921757:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304008745 00000100:00000001:2.0:1788878396.921758:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878396.921760:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.921761:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878396.921765:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878396/real 1788878396] req@ffff94e50897b800 x1875765290059264/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878396.921771:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878396.921772:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878396.921774:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878396.921775:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878396.921777:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.921778:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878396.921779:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878396.921783:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290059264/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878396.921789:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.921791:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878396.921793:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.921794:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878396.921795:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.921796:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878396.921816:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878396.921819:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878396.921822:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878396.921823:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878396.921826:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290059264:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878396.921829:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878396.921830:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878396.921832:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878396.921834:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878396.921835:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878396.921837:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878396.921840:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290059392:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878396.921842:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878396.921844:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878396.921845:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878396.921847:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878396.921849:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.921853:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878396.921854:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 00000100:00000001:2.0:1788878396.921857:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878396.921868:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878396.921873:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878396.921875:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878396.921878:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.921880:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878396.921884:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.921888:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304008201 00000100:00000001:2.0:1788878396.921891:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304008745 00000100:00000001:2.0:1788878396.921894:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304008201 00000100:00000001:2.0:1788878396.921897:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304008745 00000100:00000001:2.0:1788878396.921900:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878396.921938:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.921941:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878396.921946:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878396/real 1788878396] req@ffff94e508979180 x1875765290059392/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878396.921954:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878396.921955:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878396.921958:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878396.921960:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878396.921961:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.921962:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878396.921964:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878396.921967:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290059392/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878396.921973:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.921975:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878396.921977:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.921979:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878396.921980:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.921981:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878396.921990:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878396.921992:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878396.921995:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878396.922016:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878396.922021:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290059392:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878396.922026:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878396.922028:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878396.922030:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878396.922033:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878396.922035:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878396.922038:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878396.922042:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290059520:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878396.922047:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878396.922049:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878396.922050:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878396.922052:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878396.922055:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.922060:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878396.922061:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:2.0:1788878396.922065:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878396.922077:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878396.922081:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878396.922083:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878396.922085:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.922086:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878396.922089:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.922092:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304008202 00000100:00000001:2.0:1788878396.922095:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304008745 00000100:00000001:2.0:1788878396.922098:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304008202 00000100:00000001:2.0:1788878396.922101:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304008745 00000100:00000001:2.0:1788878396.922104:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878396.922107:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.922110:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878396.922115:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878396/real 1788878396] req@ffff94e508979c00 x1875765290059520/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878396.922158:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878396.922160:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878396.922165:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878396.922166:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878396.922168:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.922169:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878396.922171:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878396.922176:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290059520/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878396.922182:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.922184:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878396.922186:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.922187:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878396.922188:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.922190:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878396.922200:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878396.922202:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878396.922204:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878396.922206:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878396.922209:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290059520:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878396.922212:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.922214:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878396.922215:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878396.922217:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878396.922219:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878396.922221:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878396.922222:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878396.922224:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878396.922226:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878396.922229:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878396.922230:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.922232:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878396.922233:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878396.922234:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878396.922236:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878396.922237:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878396.922238:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878396.922239:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878396.922240:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878396.922242:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878396.922243:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.922244:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878396.922245:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878396.922246:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878396.922247:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878396.922248:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878396.922249:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878396.922251:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878396.922252:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878396.922253:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878396.922254:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.922255:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878396.922258:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878396.922260:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878396.922261:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878396.922263:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878402.041032:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878402.041043:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878402.041046:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878402.041050:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878402.041055:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878402.041066:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9340 00000100:00080000:3.0:1788878402.041080:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878402.041084:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878402.041086:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878402.041097:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878402.041101:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878402.041105:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878402.041107:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878402.041109:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878402.041110:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878402.041115:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878402.041117:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878402.041120:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878402.041124:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878402.041126:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041129:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878402.041131:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878402.041145:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878402.041146:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041149:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878402.041152:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878402.041157:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041170:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878402.041180:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290059648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878402.041189:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878402.041193:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041210:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878402.041212:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878402.041215:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878402.041216:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878402.041218:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878402.041220:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878402.041222:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9340 00000100:00080000:3.0:1788878402.041224:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878402.041226:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878402.041227:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878402.041230:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878402.041231:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878402.041232:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878402.041234:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878402.041236:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878402.041237:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878402.041240:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878402.041241:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878402.041243:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878402.041248:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878402.041250:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041251:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878402.041255:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878402.041257:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041259:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878402.041259:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878402.041263:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878402.041265:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788878402.041266:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041269:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878402.041270:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788878402.041271:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878402.041274:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788878402.041274:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041278:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041281:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878402.041284:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1788878402.041287:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290059776/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788878402.041288:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878402.041292:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878402.041298:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290059648:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878402.041307:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788878402.041309:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788878402.041313:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878402.041313:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878402.041317:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041318:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878402.041323:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1788878402.041324:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788878402.041329:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878402.041330:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1788878402.041332:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878402.041332:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878402.041338:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878402.041341:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9340 00000100:00080000:3.0:1788878402.041343:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878402.041344:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878402.041345:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878402.041348:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878402.041349:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878402.041351:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878402.041353:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878402.041354:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878402.041355:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878402.041359:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:3.0:1788878402.041359:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878402.041366:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878402.041367:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878402.041369:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878402.041371:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878402.041374:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878402.041375:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041377:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878402.041379:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878402.041380:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041381:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878402.041385:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878402.041386:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041387:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878402.041388:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878402.041389:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041392:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878402.041395:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290059904/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878402.041401:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878402.041402:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041404:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878402.041405:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878402.041408:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878402.041408:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878402.041409:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878402.041411:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000800:00000100:2.0:1788878402.041415:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1788878402.041416:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9340 00000100:00080000:3.0:1788878402.041422:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878402.041423:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878402.041424:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878402.041427:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878402.041428:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878402.041429:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878402.041431:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878402.041432:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878402.041433:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878402.041436:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878402.041439:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1788878402.041440:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878402.041445:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041445:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041450:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788878402.041450:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878402.041455:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878402.041456:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041457:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041458:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878402.041461:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878402.041463:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041464:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304013865 00000100:00000001:3.0:1788878402.041465:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878402.041469:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878402.041471:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041472:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878402.041473:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878402.041474:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041477:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041481:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00100000:3.0:1788878402.041481:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290060032/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878402.041489:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878402.041491:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878402.041493:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041494:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304013865 00000100:00000001:2.0:1788878402.041502:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878402.041507:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878402.041511:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041514:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878402.041521:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878402/real 1788878402] req@ffff94e50449d180 x1875765290059648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878402.041532:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878402.041534:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878402.041538:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878402.041541:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878402.041543:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041546:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878402.041548:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878402.041553:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290059648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878402.041562:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041566:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878402.041569:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041571:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878402.041573:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041576:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878402.041578:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878402.041582:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878402.041587:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878402.041590:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878402.041593:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290059648:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878402.041597:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041599:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878402.041601:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878402.041603:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878402.041606:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878402.041608:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878402.041610:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878402.041613:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878402.041614:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878402.041617:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878402.041618:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041620:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041627:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878402.041629:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041630:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878402.041631:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878402.041632:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878402.041634:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878402.041635:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878402.041637:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878402.041638:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878402.041639:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878402.041641:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290059776:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878402.041644:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878402.041646:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878402.041647:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878402.041648:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878402.041650:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041655:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878402.041656:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:2.0:1788878402.041661:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878402.041673:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878402.041681:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878402.041682:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878402.041683:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041685:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878402.041687:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041690:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304013323 00000100:00000001:2.0:1788878402.041693:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304013865 00000100:00000001:2.0:1788878402.041694:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304013323 00000100:00000001:2.0:1788878402.041696:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304013865 00000100:00000001:2.0:1788878402.041698:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878402.041699:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041700:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878402.041703:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878402/real 1788878402] req@ffff94e50449c000 x1875765290059776/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878402.041708:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878402.041708:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878402.041710:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878402.041711:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878402.041712:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041714:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878402.041715:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878402.041717:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290059776/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878402.041722:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041723:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878402.041724:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041725:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878402.041726:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041727:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878402.041741:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878402.041743:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878402.041746:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878402.041747:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878402.041750:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290059776:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878402.041753:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878402.041753:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878402.041754:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878402.041756:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878402.041757:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878402.041758:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878402.041759:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290059904:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878402.041761:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878402.041762:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878402.041763:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878402.041765:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878402.041767:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041770:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878402.041771:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:2.0:1788878402.041774:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878402.041780:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878402.041783:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878402.041784:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878402.041785:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041786:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878402.041788:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041790:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304013321 00000100:00000001:2.0:1788878402.041792:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304013865 00000100:00000001:2.0:1788878402.041794:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304013321 00000100:00000001:2.0:1788878402.041796:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304013865 00000100:00000001:2.0:1788878402.041797:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878402.041798:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041799:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878402.041802:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878402/real 1788878402] req@ffff94e50449f480 x1875765290059904/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878402.041808:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878402.041809:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878402.041810:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878402.041811:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878402.041812:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041813:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878402.041814:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878402.041817:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290059904/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878402.041822:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041823:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878402.041824:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041825:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878402.041826:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041827:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878402.041840:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878402.041841:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878402.041843:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878402.041844:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878402.041846:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290059904:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878402.041848:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878402.041849:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878402.041850:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878402.041852:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878402.041853:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878402.041855:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878402.041856:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290060032:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878402.041858:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878402.041860:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878402.041861:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878402.041862:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878402.041864:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041867:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878402.041868:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:2.0:1788878402.041870:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878402.041875:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878402.041878:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878402.041879:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878402.041880:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041881:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878402.041883:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041885:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304013321 00000100:00000001:2.0:1788878402.041887:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304013865 00000100:00000001:2.0:1788878402.041888:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304013321 00000100:00000001:2.0:1788878402.041890:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304013865 00000100:00000001:2.0:1788878402.041891:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878402.041892:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.041893:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878402.041896:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878402/real 1788878402] req@ffff94e50449c700 x1875765290060032/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878402.041930:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878402.041931:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878402.041932:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878402.041950:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878402.041951:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041953:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878402.041954:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878402.041958:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290060032/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878402.041971:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041990:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878402.041994:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041995:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878402.041996:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.041998:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878402.042013:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878402.042016:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878402.042019:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878402.042021:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878402.042025:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290060032:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878402.042031:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.042034:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878402.042035:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878402.042038:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878402.042041:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878402.042043:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878402.042044:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878402.042047:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878402.042049:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878402.042052:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878402.042054:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.042056:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878402.042058:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878402.042060:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878402.042062:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878402.042063:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878402.042065:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878402.042068:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878402.042069:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878402.042072:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878402.042074:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.042076:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878402.042078:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878402.042080:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878402.042083:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878402.042085:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878402.042087:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878402.042089:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878402.042091:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878402.042093:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878402.042094:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.042096:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878402.042099:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878402.042101:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878402.042102:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878402.042104:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878407.161103:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878407.161115:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878407.161121:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878407.161128:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878407.161138:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878407.161151:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9346 00000100:00080000:2.0:1788878407.161169:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878407.161175:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878407.161180:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878407.161193:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878407.161198:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878407.161204:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878407.161211:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878407.161216:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878407.161220:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878407.161228:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878407.161232:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878407.161236:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878407.161241:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878407.161245:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.161248:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878407.161251:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878407.161264:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878407.161267:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.161271:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878407.161275:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878407.161282:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.161297:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878407.161311:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290060160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878407.161326:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878407.161333:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.161358:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878407.161365:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878407.161374:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878407.161381:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878407.161386:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878407.161391:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878407.161394:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788878407.161401:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788878407.161407:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9346 00000100:00000001:3.0:1788878407.161409:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878407.161418:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878407.161424:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878407.161426:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1788878407.161433:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788878407.161436:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788878407.161440:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878407.161444:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:3.0:1788878407.161447:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878407.161463:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:3.0:1788878407.161464:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290060160:192.168.203.129@tcp:250:kworker.0 00000020:00000001:2.0:1788878407.161472:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788878407.161476:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1788878407.161480:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:3.0:1788878407.161485:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:2.0:1788878407.161494:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:3.0:1788878407.161494:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878407.161502:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788878407.161506:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788878407.161511:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.161513:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878407.161529:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878407.161535:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.161539:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788878407.161540:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788878407.161545:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 02000000:00000001:2.0:1788878407.161555:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878407.161563:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.161566:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878407.161572:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878407.161578:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.161583:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878407.161592:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878407.161595:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.161598:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878407.161601:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878407.161605:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.161616:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878407.161629:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290060288/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878407.161646:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878407.161651:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.161658:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878407.161664:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878407.161671:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878407.161674:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878407.161677:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878407.161681:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878407.161685:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9346 00000100:00080000:2.0:1788878407.161691:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878407.161694:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878407.161696:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878407.161701:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878407.161703:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878407.161707:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878407.161710:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878407.161714:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878407.161717:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878407.161725:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878407.161730:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878407.161734:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788878407.161743:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1788878407.161744:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878407.161755:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.161759:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878407.161764:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878407.161770:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.161774:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878407.161783:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878407.161786:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.161790:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878407.161794:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878407.161798:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.161811:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878407.161822:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290060416/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878407.161838:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878407.161843:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.161847:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878407.161853:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878407.161860:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878407.161863:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878407.161866:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878407.161872:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878407.161876:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9346 00000100:00080000:2.0:1788878407.161883:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878407.161886:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878407.161888:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878407.161892:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878407.161895:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878407.161897:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878407.161900:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878407.161980:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878407.161983:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878407.161988:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878407.161994:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878407.161997:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878407.162003:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878407.162006:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.162009:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878407.162012:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878407.162016:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.162019:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878407.162025:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878407.162027:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.162030:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878407.162032:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878407.162034:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.162042:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878407.162049:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290060544/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878407.162059:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878407.162062:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878407.162066:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788878407.162118:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878407.162140:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878407.162148:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878407.162154:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.162160:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878407.162173:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.162183:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304018986 00000100:00000001:3.0:1788878407.162201:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878407.162213:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304018986 00000100:00000001:3.0:1788878407.162225:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878407.162322:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878407.162328:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.162333:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878407.162346:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878407/real 1788878407] req@ffff94e5045e7800 x1875765290060160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878407.162369:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878407.162372:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878407.162379:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878407.162384:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878407.162388:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.162393:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878407.162397:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878407.162407:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290060160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878407.162424:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.162430:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878407.162436:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.162440:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878407.162442:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.162447:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878407.162451:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878407.162457:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878407.162466:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878407.162471:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878407.162480:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290060160:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878407.162490:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.162495:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878407.162498:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878407.162503:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878407.162510:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878407.162513:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878407.162517:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878407.162523:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878407.162526:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878407.162532:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878407.162535:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.162539:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.162552:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878407.162556:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.162560:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878407.162563:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878407.162566:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878407.162569:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878407.162571:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878407.162576:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878407.162580:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878407.162585:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878407.162593:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290060288:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878407.162602:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878407.162607:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878407.162610:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878407.162614:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878407.162620:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.162633:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878407.162636:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 00000100:00000001:3.0:1788878407.162644:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878407.162678:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878407.162687:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878407.162693:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878407.162697:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.162701:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878407.162706:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.162715:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304018444 00000100:00000001:3.0:1788878407.162722:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304018986 00000100:00000001:3.0:1788878407.162726:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304018444 00000100:00000001:3.0:1788878407.162731:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304018986 00000100:00000001:3.0:1788878407.162736:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878407.162740:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.162744:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878407.162753:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878407/real 1788878407] req@ffff94e5045e6300 x1875765290060288/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878407.162773:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878407.162776:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878407.162781:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878407.162785:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878407.162788:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.162793:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878407.162795:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878407.162804:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290060288/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878407.162820:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.162825:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878407.162830:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.162834:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878407.162836:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.162840:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878407.162862:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878407.162869:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878407.162876:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878407.162881:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878407.162890:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290060288:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878407.162898:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878407.162901:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878407.162948:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878407.162980:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878407.162985:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878407.162991:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878407.163000:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290060416:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878407.163008:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878407.163013:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878407.163016:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878407.163020:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878407.163026:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.163039:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878407.163042:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:3.0:1788878407.163048:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878407.163077:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878407.163089:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878407.163094:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878407.163097:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.163100:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878407.163107:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.163114:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304018443 00000100:00000001:3.0:1788878407.163120:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304018986 00000100:00000001:3.0:1788878407.163124:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304018443 00000100:00000001:3.0:1788878407.163128:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304018986 00000100:00000001:3.0:1788878407.163132:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878407.163137:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.163141:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878407.163150:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878407/real 1788878407] req@ffff94e5045e5500 x1875765290060416/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878407.163182:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878407.163184:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878407.163190:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878407.163194:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878407.163197:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.163201:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878407.163203:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878407.163213:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290060416/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878407.163229:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.163233:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878407.163238:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.163241:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878407.163245:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.163249:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878407.163269:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878407.163276:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878407.163284:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878407.163288:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878407.163296:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290060416:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878407.163324:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878407.163327:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878407.163330:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878407.163342:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878407.163346:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878407.163350:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878407.163357:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290060544:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878407.163364:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878407.163368:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878407.163373:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878407.163376:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878407.163383:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.163394:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878407.163397:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:3.0:1788878407.163405:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878407.163435:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878407.163445:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878407.163450:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878407.163453:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.163457:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878407.163463:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.163470:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304018443 00000100:00000001:3.0:1788878407.163475:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304018986 00000100:00000001:3.0:1788878407.163480:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304018443 00000100:00000001:3.0:1788878407.163484:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304018986 00000100:00000001:3.0:1788878407.163488:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878407.163493:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.163497:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878407.163506:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878407/real 1788878407] req@ffff94e5045e5c00 x1875765290060544/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878407.163524:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878407.163526:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878407.163532:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878407.163535:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878407.163538:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.163541:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878407.163544:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878407.163552:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290060544/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878407.163567:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.163572:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878407.163578:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.163581:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878407.163584:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.163588:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878407.163610:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878407.163616:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878407.163622:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878407.163627:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878407.163635:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290060544:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878407.163643:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.163647:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878407.163650:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878407.163654:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878407.163659:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878407.163662:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878407.163665:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878407.163669:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878407.163672:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878407.163677:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878407.163679:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.163683:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878407.163686:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878407.163688:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878407.163691:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878407.163693:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878407.163695:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878407.163699:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878407.163702:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878407.163705:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878407.163707:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.163711:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878407.163713:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878407.163716:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878407.163719:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878407.163721:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878407.163723:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878407.163726:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878407.163729:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878407.163732:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878407.163735:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.163739:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878407.163746:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878407.163749:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878407.163753:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878407.163756:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878412.281054:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878412.281065:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878412.281068:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878412.281073:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878412.281080:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878412.281090:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9351 00000100:00080000:2.0:1788878412.281104:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878412.281108:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878412.281111:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878412.281121:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878412.281125:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878412.281129:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878412.281132:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878412.281136:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878412.281138:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878412.281144:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878412.281148:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878412.281151:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878412.281155:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878412.281158:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878412.281161:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878412.281165:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878412.281178:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878412.281180:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878412.281184:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878412.281187:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878412.281192:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878412.281207:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878412.281218:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4380 x1875765290060672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878412.281233:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878412.281238:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878412.281256:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878412.281260:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878412.281264:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878412.281266:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878412.281268:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878412.281272:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878412.281275:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9351 00000100:00080000:2.0:1788878412.281281:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878412.281285:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878412.281289:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878412.281294:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878412.281296:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878412.281300:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878412.281300:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878412.281306:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788878412.281308:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878412.281310:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878412.281315:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878412.281316:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878412.281319:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878412.281321:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:3.0:1788878412.281326:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1788878412.281332:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:3.0:1788878412.281332:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878412.281337:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878412.281341:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878412.281346:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290060672:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878412.281349:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788878412.281356:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878412.281359:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878412.281360:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878412.281366:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878412.281367:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878412.281370:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788878412.281374:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788878412.281377:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788878412.281380:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878412.281387:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878412.281390:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.281396:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788878412.281397:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788878412.281400:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074549120 : -117763635002496 : ffff94e5045e4380) 02000000:00000001:2.0:1788878412.281402:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.281407:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878412.281409:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878412.281413:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878412.281416:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878412.281425:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878412.281434:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5f80 x1875765290060800/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878412.281446:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878412.281451:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788878412.281453:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878412.281473:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.281476:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788878412.281478:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788878412.281483:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788878412.281485:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878412.281487:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878412.281491:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788878412.281492:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:2.0:1788878412.281498:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878412.281500:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878412.281508:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788878412.281509:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304024105 00000100:00080000:2.0:1788878412.281516:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9351 00000100:00000001:3.0:1788878412.281522:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1788878412.281523:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878412.281531:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878412.281532:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304024105 00000100:00000001:2.0:1788878412.281534:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878412.281538:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1788878412.281545:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788878412.281546:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1788878412.281550:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788878412.281552:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878412.281557:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878412.281557:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878412.281566:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878412/real 1788878412] req@ffff94e5045e4380 x1875765290060672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878412.281585:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1788878412.281587:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000400:00000001:3.0:1788878412.281589:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878412.281594:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878412.281595:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878412.281599:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878412.281600:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878412.281602:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.281608:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878412.281612:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878412.281615:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1788878412.281620:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4380 x1875765290060672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878412.281622:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878412.281630:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788878412.281636:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878412.281637:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878412.281641:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788878412.281646:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.281647:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.281653:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878412.281655:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878412.281656:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1788878412.281662:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788878412.281662:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878412.281666:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:2.0:1788878412.281671:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878412.281672:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788878412.281678:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878412.281680:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878412.281684:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1788878412.281688:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1788878412.281689:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290060672:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878412.281694:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.281697:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878412.281701:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788878412.281702:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878412.281707:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878412.281708:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878412.281710:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878412.281713:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878412.281715:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878412.281720:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878412.281723:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:2.0:1788878412.281724:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878412.281727:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878412.281731:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878412.281734:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878412.281737:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878412.281738:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290060928/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878412.281751:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878412.281752:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878412.281754:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878412.281761:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878412.281764:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788878412.281765:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878412.281770:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878412.281772:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878412.281778:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788878412.281778:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878412.281784:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788878412.281785:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878412.281791:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878412.281792:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788878412.281796:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788878412.281803:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9351 00000100:00000001:3.0:1788878412.281803:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878412.281811:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1788878412.281813:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1788878412.281816:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878412.281819:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788878412.281824:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878412.281825:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:3.0:1788878412.281836:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290060800:192.168.203.129@tcp:38:kworker.0 00000020:00000001:2.0:1788878412.281837:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878412.281841:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788878412.281845:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1788878412.281847:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:3.0:1788878412.281849:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878412.281852:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878412.281855:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:3.0:1788878412.281856:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878412.281862:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878412.281863:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878412.281868:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878412.281872:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878412.281875:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074556288 : -117763634995328 : ffff94e5045e5f80) 00000100:00080000:2.0:1788878412.281879:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878412.281882:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878412.281885:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878412.281890:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878412.281894:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878412.281898:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788878412.281953:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878412.281964:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878412.281969:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00100000:2.0:1788878412.281970:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788878412.281974:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00000001:2.0:1788878412.281978:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.281979:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878412.281986:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.281986:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878412.281993:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878412.281998:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788878412.282000:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304023564 02000000:00000001:2.0:1788878412.282002:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.282007:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304024105 00000100:00000001:2.0:1788878412.282009:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788878412.282011:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304023564 02000000:00000001:2.0:1788878412.282014:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788878412.282016:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304024105 02000000:00000001:2.0:1788878412.282020:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.282021:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878412.282029:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.282032:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878412.282036:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878412.282038:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878412/real 1788878412] req@ffff94e5045e5f80 x1875765290060800/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878412.282050:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290061056/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788878412.282070:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878412.282074:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878412.282081:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878412.282082:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878412.282088:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878412.282089:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.282090:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282097:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878412.282099:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878412.282101:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878412.282106:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5f80 x1875765290060800/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878412.282115:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282117:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878412.282120:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282122:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878412.282123:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282125:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878412.282139:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878412.282141:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878412.282151:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878412.282153:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878412.282156:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290060800:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878412.282160:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878412.282162:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878412.282163:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878412.282165:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878412.282167:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878412.282169:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878412.282172:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290060928:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878412.282177:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878412.282179:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878412.282180:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878412.282182:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878412.282185:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.282191:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878412.282192:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:3.0:1788878412.282195:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878412.282207:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878412.282210:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878412.282213:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878412.282214:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.282216:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878412.282218:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.282221:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304023562 00000100:00000001:3.0:1788878412.282223:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304024105 00000100:00000001:3.0:1788878412.282225:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304023562 00000100:00000001:3.0:1788878412.282227:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304024105 00000100:00000001:3.0:1788878412.282229:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878412.282231:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.282232:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878412.282236:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878412/real 1788878412] req@ffff94e5045e5c00 x1875765290060928/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878412.282242:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878412.282244:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878412.282246:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878412.282247:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878412.282248:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282250:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878412.282251:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878412.282254:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290060928/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878412.282266:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282268:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878412.282270:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282271:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878412.282273:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282274:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878412.282283:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878412.282285:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878412.282288:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878412.282290:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878412.282292:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290060928:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878412.282296:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282297:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878412.282299:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878412.282300:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878412.282302:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878412.282303:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878412.282304:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878412.282306:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878412.282307:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878412.282310:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878412.282311:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282312:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878412.282313:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878412.282314:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878412.282316:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878412.282317:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878412.282318:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878412.282319:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878412.282320:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878412.282321:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878412.282323:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282324:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282327:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878412.282329:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.282331:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878412.282332:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878412.282333:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878412.282335:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878412.282336:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878412.282338:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878412.282339:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878412.282341:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878412.282343:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290061056:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878412.282346:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878412.282347:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878412.282349:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878412.282350:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878412.282352:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.282356:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878412.282358:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:3.0:1788878412.282361:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878412.282369:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878412.282372:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878412.282374:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878412.282375:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.282377:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878412.282379:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.282382:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304023562 00000100:00000001:3.0:1788878412.282384:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304024105 00000100:00000001:3.0:1788878412.282386:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304023562 00000100:00000001:3.0:1788878412.282388:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304024105 00000100:00000001:3.0:1788878412.282390:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878412.282391:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.282393:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878412.282396:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878412/real 1788878412] req@ffff94e5045e5500 x1875765290061056/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878412.282408:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878412.282409:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878412.282411:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878412.282413:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878412.282414:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282415:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878412.282416:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878412.282419:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290061056/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878412.282425:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282426:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878412.282428:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282430:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878412.282431:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282433:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878412.282442:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878412.282444:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878412.282447:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878412.282449:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878412.282451:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290061056:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878412.282455:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282456:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878412.282457:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878412.282458:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878412.282460:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878412.282461:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878412.282465:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878412.282467:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878412.282468:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878412.282469:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878412.282471:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282472:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878412.282474:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878412.282476:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878412.282477:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878412.282479:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878417.401161:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878417.401181:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878417.401186:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878417.401194:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878417.401204:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878417.401219:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9356 00000100:00080000:3.0:1788878417.401237:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878417.401242:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878417.401245:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878417.401257:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878417.401261:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878417.401267:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878417.401270:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878417.401273:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878417.401275:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878417.401282:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878417.401285:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878417.401290:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878417.401295:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878417.401299:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.401302:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878417.401306:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878417.401321:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878417.401323:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.401328:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878417.401332:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878417.401339:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.401369:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878417.401384:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290061184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878417.401400:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878417.401407:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.401438:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878417.401443:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878417.401448:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878417.401450:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878417.401452:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878417.401457:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878417.401461:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9356 00000100:00080000:3.0:1788878417.401465:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878417.401468:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878417.401469:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878417.401474:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878417.401476:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878417.401479:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878417.401482:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878417.401485:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878417.401486:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878417.401490:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878417.401493:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878417.401495:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878417.401504:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878417.401507:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.401509:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878417.401515:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878417.401531:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.401536:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878417.401544:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878417.401548:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.401553:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878417.401556:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878417.401561:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.401571:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.401577:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1788878417.401584:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290061312/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878417.401593:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878417.401603:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878417.401609:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878417.401614:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878417.401618:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.401626:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878417.401634:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878417.401641:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878417.401645:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788878417.401648:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1788878417.401661:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878417.401662:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878417.401671:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878417.401676:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878417.401686:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00100000:2.0:1788878417.401687:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290061184:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1788878417.401698:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9356 00000100:00000001:2.0:1788878417.401711:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788878417.401716:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1788878417.401719:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878417.401731:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1788878417.401732:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.401737:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788878417.401740:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1788878417.401752:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878417.401754:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878417.401759:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 02000000:00000001:2.0:1788878417.401760:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878417.401779:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878417.401786:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878417.401788:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878417.401795:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878417.401797:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878417.401799:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.401802:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 02000000:00000001:3.0:1788878417.401804:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878417.401814:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878417.401815:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788878417.401827:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.401831:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878417.401836:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878417.401841:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.401843:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878417.401851:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878417.401853:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.401855:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878417.401857:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878417.401861:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.401868:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878417.401876:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290061440/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878417.401888:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878417.401891:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.401895:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878417.402000:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788878417.402008:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878417.402023:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878417.402029:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878417.402036:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878417.402040:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304029226 00000100:00000001:3.0:1788878417.402046:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878417.402062:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9356 00000100:00000001:2.0:1788878417.402063:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788878417.402096:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878417.402099:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304029226 00000100:00000001:3.0:1788878417.402105:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878417.402109:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878417.402110:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878417.402124:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878417.402129:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878417.402129:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878417.402140:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788878417.402141:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.402147:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788878417.402148:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878417.402156:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878417.402160:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000001:3.0:1788878417.402160:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878417.402170:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878417.402176:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878417.402176:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878417.402191:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878417.402192:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878417.402198:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878417.402200:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878417.402207:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1788878417.402216:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:2.0:1788878417.402217:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878417.402224:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.402230:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878417.402238:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290061312:192.168.203.129@tcp:38:kworker.0 02000000:00000001:3.0:1788878417.402239:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878417.402252:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878417.402255:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878417.402264:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878417.402265:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878417.402273:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.402273:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878417.402282:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788878417.402283:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1788878417.402291:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.402295:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878417.402296:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878417.402303:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878417.402306:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.402309:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878417.402312:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878417.402314:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788878417.402316:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.402322:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:3.0:1788878417.402329:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.402335:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1788878417.402343:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290061568/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878417.402355:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878417.402359:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878417.402366:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.402388:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.402400:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304028684 00000100:00000001:2.0:1788878417.402406:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304029226 00000100:00000001:2.0:1788878417.402411:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304028684 00000100:00000001:2.0:1788878417.402416:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304029226 00000100:00000001:2.0:1788878417.402422:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878417.402427:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.402432:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878417.402435:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878417.402437:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878417.402442:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878417.402463:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878417.402466:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878417.402472:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290061440:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878417.402477:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878417.402479:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878417.402481:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878417.402482:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878417.402486:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.402492:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878417.402493:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:2.0:1788878417.402497:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878417.402508:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.402511:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304028682 00000100:00000001:2.0:1788878417.402513:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304029226 00000100:00000001:2.0:1788878417.402516:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304028682 00000100:00000001:2.0:1788878417.402518:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304029226 00000100:00000001:2.0:1788878417.402521:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878417.402522:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.402525:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.402527:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878417.402532:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878417.402533:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788878417.402536:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878417.402539:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878417.402540:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878417.402543:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878417.402545:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878417.402548:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878417.402550:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878417.402553:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878417.402558:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290061568:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878417.402563:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878417.402566:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878417.402568:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878417.402570:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878417.402575:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.402582:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878417.402584:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:2.0:1788878417.402588:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878417.402603:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.402607:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304028682 00000100:00000001:2.0:1788878417.402609:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304029226 00000100:00000001:2.0:1788878417.402611:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304028682 00000100:00000001:2.0:1788878417.402613:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304029226 00000100:00000001:2.0:1788878417.402615:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878417.402617:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.402620:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.402622:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878417.402624:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878417.402626:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788878417.402628:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878417.402629:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878417.402630:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878417.402632:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878420.473458:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878420.473467:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878420.473482:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878420.473497:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878417/real 1788878420] req@ffff94e50449ce00 x1875765290061568/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878420.473515:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878420.473518:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878420.473527:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878420.473531:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878420.473533:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.473537:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878420.473539:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878420.473545:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290061568/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878420.473555:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.473560:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878420.473566:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.473569:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878420.473571:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.473576:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878420.473599:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878420.473606:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878420.473615:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878420.473620:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878420.473627:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290061568:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878420.473637:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878420.473644:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878417/real 1788878420] req@ffff94e50449d880 x1875765290061312/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878420.473662:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878420.473665:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878420.473671:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878420.473675:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878420.473677:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.473681:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878420.473683:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878420.473691:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290061312/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878420.473705:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.473710:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878420.473714:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.473717:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878420.473720:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.473724:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878420.473744:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878420.473749:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878420.473757:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878420.473761:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878420.473767:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290061312:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878420.473775:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878420.473782:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878417/real 1788878420] req@ffff94e50449c380 x1875765290061440/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878420.473798:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878420.473799:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878420.473805:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878420.473809:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878420.473811:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.473815:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878420.473817:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878420.473825:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290061440/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878420.473839:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.473843:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878420.473847:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.473850:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878420.473852:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.473856:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878420.473953:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878420.473961:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878420.473969:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878420.473973:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878420.473979:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290061440:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878420.473986:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878420.473994:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878417/real 1788878420] req@ffff94e50449fb80 x1875765290061184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878420.474009:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878420.474011:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878420.474018:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878420.474021:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878420.474023:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.474026:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878420.474029:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878420.474038:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290061184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878420.474050:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.474056:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878420.474060:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.474063:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878420.474066:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.474069:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878420.474072:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878420.474078:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878420.474084:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878420.474088:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878420.474095:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290061184:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878420.474103:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.474107:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878420.474110:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878420.474115:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878420.474123:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878420.474139:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878420.474143:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878420.474149:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878420.474152:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878420.474158:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878420.474161:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.474165:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878420.474167:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878420.474171:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878420.474175:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878420.474177:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878420.474180:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878420.474183:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878420.474185:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878420.474188:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878420.474190:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.474193:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878420.474195:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878420.474198:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878420.474201:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878420.474203:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878420.474205:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878420.474209:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878420.474211:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878420.474214:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878420.474216:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.474219:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878420.474221:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878420.474223:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878420.474226:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878420.474228:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878420.474231:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878420.474233:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878420.474236:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878420.474239:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878420.474242:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.474246:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878420.474257:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878420.474261:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878420.474265:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878420.474267:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878422.520986:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878422.520994:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878422.520997:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878422.520999:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878422.521003:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878422.521008:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9361 00000100:00080000:3.0:1788878422.521018:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878422.521021:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878422.521022:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878422.521028:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878422.521030:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878422.521033:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878422.521034:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878422.521036:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878422.521037:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878422.521041:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878422.521043:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878422.521045:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878422.521048:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878422.521050:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878422.521051:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878422.521053:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878422.521061:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878422.521062:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878422.521064:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878422.521066:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878422.521070:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878422.521084:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878422.521091:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290061696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878422.521099:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878422.521103:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878422.521115:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878422.521117:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878422.521120:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878422.521121:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878422.521122:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878422.521124:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878422.521126:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9361 00000100:00080000:3.0:1788878422.521128:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878422.521130:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878422.521131:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878422.521133:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878422.521134:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878422.521135:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878422.521137:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878422.521138:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878422.521140:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878422.521142:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878422.521145:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878422.521147:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878422.521155:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878422.521157:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521158:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878422.521160:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878422.521165:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878422.521167:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878422.521171:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00000001:3.0:1788878422.521171:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521174:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878422.521176:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521177:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878422.521183:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878422.521184:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878422.521186:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878422.521188:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878422.521189:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878422.521192:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1788878422.521196:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290061696:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788878422.521197:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878422.521201:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521202:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878422.521206:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878422.521209:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878422.521211:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878422.521211:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878422.521217:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878422.521222:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290061824/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878422.521232:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878422.521235:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521238:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878422.521239:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521242:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00080000:3.0:1788878422.521244:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788878422.521250:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788878422.521251:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878422.521255:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878422.521256:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878422.521260:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878422.521263:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9361 00000100:00080000:3.0:1788878422.521276:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878422.521279:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878422.521282:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878422.521285:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878422.521288:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878422.521292:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000800:00000100:2.0:1788878422.521293:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:3.0:1788878422.521297:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878422.521301:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878422.521303:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878422.521308:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878422.521309:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878422.521311:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521315:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788878422.521316:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878422.521321:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878422.521322:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878422.521327:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788878422.521328:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878422.521332:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878422.521335:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788878422.521336:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878422.521342:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521343:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304034345 00000100:00000001:3.0:1788878422.521344:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878422.521353:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878422.521355:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878422.521357:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878422.521358:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788878422.521359:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878422.521363:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521368:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304034345 00000100:00000001:3.0:1788878422.521371:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521376:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00100000:3.0:1788878422.521380:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290061952/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878422.521383:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878422.521389:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521394:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788878422.521396:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878422.521400:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878422.521404:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878422/real 1788878422] req@ffff94e50449df80 x1875765290061696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878422.521405:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878422.521410:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878422.521420:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878422.521424:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878422.521427:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:2.0:1788878422.521430:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788878422.521431:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000400:00000001:2.0:1788878422.521434:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1788878422.521437:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9361 00000100:00000001:2.0:1788878422.521441:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:3.0:1788878422.521443:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878422.521458:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878422.521459:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878422.521462:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878422.521463:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878422.521469:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1788878422.521469:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878422.521473:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788878422.521474:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788878422.521476:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878422.521480:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:2.0:1788878422.521484:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290061696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878422.521484:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878422.521488:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878422.521496:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878422.521503:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521504:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878422.521506:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878422.521510:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878422.521511:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878422.521518:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878422.521518:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521524:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878422.521525:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878422.521527:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1788878422.521529:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878422.521532:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000008:00000001:3.0:1788878422.521534:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521536:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878422.521538:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878422.521542:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1788878422.521545:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878422.521549:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878422.521551:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878422.521552:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878422.521555:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878422.521557:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1788878422.521559:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878422.521566:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290061696:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878422.521568:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521577:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878422.521579:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290062080/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878422.521582:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878422.521585:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878422.521590:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788878422.521593:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788878422.521596:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1788878422.521597:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878422.521601:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:3.0:1788878422.521603:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878422.521605:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878422.521611:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878422.521613:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878422.521616:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878422.521618:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.521621:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.521629:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878422.521631:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521634:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878422.521635:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878422.521637:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878422.521638:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878422.521640:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878422.521642:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878422.521644:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878422.521646:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878422.521651:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290061824:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878422.521654:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878422.521657:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878422.521658:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878422.521660:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878422.521663:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521670:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878422.521672:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000100:00000001:2.0:1788878422.521676:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878422.521692:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878422.521697:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878422.521700:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878422.521701:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521703:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878422.521706:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521717:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304033803 00000100:00000001:2.0:1788878422.521720:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304034345 00000100:00000001:2.0:1788878422.521723:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304033803 00000100:00000001:2.0:1788878422.521725:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304034345 00000100:00000001:2.0:1788878422.521727:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878422.521729:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521731:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878422.521737:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878422/real 1788878422] req@ffff94e50449e680 x1875765290061824/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878422.521746:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878422.521747:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878422.521749:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878422.521752:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878422.521754:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.521756:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878422.521758:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878422.521761:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290061824/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878422.521768:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.521770:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878422.521773:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.521775:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878422.521776:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.521778:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878422.521793:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878422.521797:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878422.521802:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878422.521805:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878422.521809:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290061824:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878422.521818:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878422.521820:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878422.521822:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878422.521824:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878422.521826:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878422.521829:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878422.521832:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290061952:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878422.521835:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878422.521837:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878422.521839:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878422.521841:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878422.521844:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521849:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878422.521850:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:2.0:1788878422.521855:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878422.521881:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878422.521886:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878422.521888:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878422.521890:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521892:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878422.521895:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521898:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304033801 00000100:00000001:2.0:1788878422.521933:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304034345 00000100:00000001:2.0:1788878422.521937:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304033802 00000100:00000001:2.0:1788878422.521939:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304034345 00000100:00000001:2.0:1788878422.521941:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878422.521943:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.521946:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878422.521950:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878422/real 1788878422] req@ffff94e50449fb80 x1875765290061952/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878422.521959:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878422.521961:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878422.521965:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878422.521966:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878422.521967:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.521970:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878422.521971:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878422.521975:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290061952/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878422.521982:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.521984:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878422.521987:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.521988:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878422.521990:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.521991:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878422.522006:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878422.522009:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878422.522012:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878422.522014:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878422.522018:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290061952:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878422.522022:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878422.522023:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878422.522025:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878422.522027:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878422.522029:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878422.522031:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878422.522034:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290062080:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878422.522037:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878422.522039:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878422.522040:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878422.522042:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878422.522045:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.522050:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878422.522052:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:2.0:1788878422.522055:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878422.522067:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878422.522072:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878422.522074:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878422.522076:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.522077:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878422.522081:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.522084:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304033802 00000100:00000001:2.0:1788878422.522087:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304034345 00000100:00000001:2.0:1788878422.522089:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304033802 00000100:00000001:2.0:1788878422.522091:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304034345 00000100:00000001:2.0:1788878422.522094:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878422.522096:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.522098:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878422.522101:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878422/real 1788878422] req@ffff94e50449c380 x1875765290062080/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878422.522109:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878422.522110:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878422.522113:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878422.522115:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878422.522116:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.522118:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878422.522120:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878422.522123:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290062080/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878422.522130:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.522132:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878422.522135:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.522137:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878422.522138:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.522140:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878422.522154:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878422.522157:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878422.522160:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878422.522162:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878422.522165:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290062080:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878422.522169:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.522171:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878422.522173:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878422.522174:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878422.522177:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878422.522178:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878422.522180:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878422.522182:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878422.522184:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878422.522186:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878422.522187:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.522189:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878422.522190:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878422.522192:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878422.522193:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878422.522194:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878422.522196:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878422.522197:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878422.522199:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878422.522200:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878422.522202:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.522203:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878422.522204:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878422.522206:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878422.522207:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878422.522208:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878422.522209:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878422.522211:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878422.522212:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878422.522214:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878422.522215:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.522216:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878422.522220:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878422.522223:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878422.522225:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878422.522226:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878427.641073:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878427.641102:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878427.641106:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878427.641112:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878427.641119:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878427.641129:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9366 00000100:00080000:2.0:1788878427.641144:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878427.641148:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878427.641151:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878427.641163:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878427.641167:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878427.641172:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878427.641176:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878427.641180:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878427.641183:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878427.641190:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878427.641193:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878427.641197:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878427.641201:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878427.641204:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878427.641207:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878427.641209:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878427.641221:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878427.641223:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878427.641226:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878427.641229:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878427.641234:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878427.641247:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878427.641275:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290062208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878427.641294:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878427.641299:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878427.641321:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878427.641341:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878427.641345:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878427.641348:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878427.641350:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878427.641353:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878427.641356:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9366 00000100:00080000:2.0:1788878427.641360:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878427.641363:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878427.641365:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878427.641369:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878427.641371:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878427.641374:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878427.641378:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878427.641381:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878427.641383:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878427.641387:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878427.641407:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878427.641409:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878427.641416:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788878427.641416:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878427.641423:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878427.641426:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878427.641429:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878427.641436:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878427.641436:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878427.641444:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878427.641446:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878427.641452:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878427.641454:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:3.0:1788878427.641464:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290062208:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1788878427.641465:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788878427.641494:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1788878427.641498:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878427.641501:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788878427.641508:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878427.641509:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878427.641517:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878427.641524:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878427.641526:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878427.641532:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878427.641554:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878427.641557:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788878427.641561:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788878427.641562:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.641566:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 00000100:00000001:3.0:1788878427.641573:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878427.641580:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878427.641592:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290062336/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878427.641602:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878427.641622:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878427.641627:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878427.641630:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788878427.641637:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788878427.641639:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878427.641643:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878427.641645:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878427.641649:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788878427.641652:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788878427.641656:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9366 00000100:00000001:3.0:1788878427.641660:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788878427.641664:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878427.641665:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878427.641672:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878427.641673:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878427.641679:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878427.641685:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878427.641686:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878427.641692:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788878427.641695:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304039465 00000020:00000001:2.0:1788878427.641696:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878427.641705:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878427.641708:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878427.641710:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878427.641713:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878427.641716:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878427.641717:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878427.641722:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878427.641744:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878427.641747:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878427.641749:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878427.641751:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878427.641755:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878427.641757:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.641763:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304039465 02000000:00000001:2.0:1788878427.641764:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878427.641768:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.641779:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878427.641785:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788878427.641786:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878427.641791:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788878427.641793:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878427.641798:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.641799:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878427.641812:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878427.641813:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878427/real 1788878427] req@ffff94e5045e6300 x1875765290062208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878427.641845:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290062464/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788878427.641859:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878427.641864:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878427.641869:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878427.641889:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878427.641891:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878427.641893:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878427.641899:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.641899:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878427.641902:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878427.641905:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878427.641912:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788878427.641913:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290062208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878427.641948:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.641952:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878427.641956:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.641962:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1788878427.641964:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878427.641965:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878427.641972:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788878427.641972:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1788878427.641978:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878427.641979:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878427.641988:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788878427.641990:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878427.641995:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00080000:2.0:1788878427.641999:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9366 00000100:00000001:3.0:1788878427.642002:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:2.0:1788878427.642007:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788878427.642010:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290062208:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878427.642016:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878427.642031:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878427.642032:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878427.642036:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1788878427.642039:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788878427.642040:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000020:00000001:2.0:1788878427.642046:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:3.0:1788878427.642047:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:2.0:1788878427.642054:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:3.0:1788878427.642055:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878427.642059:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878427.642061:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878427.642063:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:2.0:1788878427.642069:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788878427.642089:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1788878427.642091:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:3.0:1788878427.642093:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878427.642102:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878427.642106:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878427.642107:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878427.642110:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878427.642121:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642123:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1788878427.642132:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878427.642140:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878427.642141:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642150:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788878427.642155:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642156:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878427.642159:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878427.642164:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878427.642167:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878427.642168:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878427.642195:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:2.0:1788878427.642196:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788878427.642198:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878427.642206:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788878427.642209:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878427.642214:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290062336:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878427.642219:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642221:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878427.642226:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878427.642228:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878427.642229:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878427.642231:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878427.642235:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878427.642238:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878427.642240:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878427.642244:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878427.642248:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642250:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878427.642254:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000100:00000001:2.0:1788878427.642258:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642276:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1788878427.642287:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290062592/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878427.642297:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000100:3.0:1788878427.642303:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878427.642304:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878427.642314:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642314:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878427.642320:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878427.642345:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642349:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878427.642355:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642360:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304038924 00000100:00000001:3.0:1788878427.642364:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304039465 00000100:00000001:3.0:1788878427.642367:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304038924 00000100:00000001:3.0:1788878427.642370:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304039465 00000100:00000001:3.0:1788878427.642373:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878427.642375:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642377:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878427.642383:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878427/real 1788878427] req@ffff94e5045e7800 x1875765290062336/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878427.642392:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878427.642411:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878427.642414:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878427.642416:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878427.642418:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642420:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878427.642422:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878427.642427:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290062336/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878427.642446:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642449:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878427.642452:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642454:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878427.642456:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642458:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878427.642471:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878427.642474:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878427.642478:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878427.642481:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878427.642485:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290062336:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878427.642489:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878427.642491:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878427.642493:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878427.642496:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878427.642499:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878427.642501:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878427.642505:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290062464:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878427.642508:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878427.642529:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878427.642530:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878427.642532:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878427.642535:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642540:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878427.642542:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:3.0:1788878427.642545:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878427.642557:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878427.642561:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878427.642563:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878427.642565:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642567:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878427.642570:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642573:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304038922 00000100:00000001:3.0:1788878427.642576:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304039465 00000100:00000001:3.0:1788878427.642578:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304038922 00000100:00000001:3.0:1788878427.642580:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304039465 00000100:00000001:3.0:1788878427.642582:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878427.642584:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642586:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878427.642590:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878427/real 1788878427] req@ffff94e5045e7100 x1875765290062464/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878427.642616:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878427.642618:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878427.642621:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878427.642622:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878427.642624:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642626:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878427.642628:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878427.642632:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290062464/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878427.642644:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642647:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878427.642650:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642652:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878427.642653:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642656:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878427.642686:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878427.642690:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878427.642695:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878427.642698:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878427.642704:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290062464:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878427.642710:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642712:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878427.642714:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878427.642717:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878427.642721:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878427.642723:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878427.642725:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878427.642730:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878427.642732:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878427.642735:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878427.642737:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642739:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878427.642741:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878427.642743:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878427.642745:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878427.642747:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878427.642749:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878427.642751:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878427.642754:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878427.642756:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878427.642758:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642760:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642765:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878427.642767:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642770:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878427.642772:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878427.642775:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878427.642777:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878427.642779:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878427.642782:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878427.642784:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878427.642786:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878427.642789:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290062592:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878427.642792:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878427.642794:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878427.642796:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878427.642798:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878427.642801:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642808:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878427.642809:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:3.0:1788878427.642813:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878427.642825:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878427.642852:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878427.642855:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878427.642856:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642858:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878427.642861:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642865:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304038922 00000100:00000001:3.0:1788878427.642879:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304039465 00000100:00000001:3.0:1788878427.642881:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304038922 00000100:00000001:3.0:1788878427.642883:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304039465 00000100:00000001:3.0:1788878427.642885:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878427.642887:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.642889:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878427.642893:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878427/real 1788878427] req@ffff94e5045e4a80 x1875765290062592/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878427.642922:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878427.642924:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878427.642926:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878427.642928:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878427.642930:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642932:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878427.642933:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878427.642937:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290062592/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878427.642944:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642946:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878427.642949:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642950:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878427.642952:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642954:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878427.642964:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878427.642967:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878427.642970:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878427.642972:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878427.642975:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290062592:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878427.642978:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642980:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878427.642981:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878427.642982:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878427.642984:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878427.642986:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878427.642987:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878427.642989:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878427.642990:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878427.642992:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878427.642993:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642995:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878427.642998:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878427.642999:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878427.643001:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878427.643003:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.761237:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878432.761244:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878432.761248:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878432.761252:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878432.761257:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878432.761265:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878432.761271:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878432.761276:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878432.761297:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290062720:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878432.761303:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878432.761310:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878432.761314:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878432.761318:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878432.761325:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.761343:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878432.761347:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286519680 : -117763423031936 : ffff94e51100ad80) 00000100:00000001:3.0:1788878432.761358:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878432.761403:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878432.761421:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878432.761428:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878432.761433:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.761437:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878432.761444:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.761452:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304044585 00000100:00000001:3.0:1788878432.761467:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878432.761484:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304044585 00000100:00000001:3.0:1788878432.761492:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878432.761500:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878432.761506:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.761513:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878432.761524:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878432/real 1788878432] req@ffff94e51100ad80 x1875765290062720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878432.761552:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878432.761558:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878432.761568:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878432.761575:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878432.761582:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.761590:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878432.761596:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878432.761612:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100ad80 x1875765290062720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878432.761649:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.761654:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878432.761659:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.761665:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878432.761669:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.761675:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878432.761678:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878432.761685:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878432.761693:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878432.761697:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878432.761701:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290062720:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878432.761707:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.761710:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878432.761712:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878432.761714:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878432.761718:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878432.761720:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878432.761722:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878432.761725:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878432.761726:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878432.761729:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878432.761730:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.761733:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.761739:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878432.761742:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.761744:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878432.761747:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878432.761749:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878432.761750:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878432.761752:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878432.761755:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878432.761758:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878432.761761:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878432.761765:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290062848:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878432.761770:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878432.761774:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878432.761776:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878432.761778:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878432.761782:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.761807:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878432.761810:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508032 : -117763423043584 : ffff94e511008000) 00000100:00000001:3.0:1788878432.761816:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878432.761833:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878432.761838:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878432.761841:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878432.761842:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.761844:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878432.761846:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.761850:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304044043 00000100:00000001:3.0:1788878432.761853:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304044585 00000100:00000001:3.0:1788878432.761855:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304044043 00000100:00000001:3.0:1788878432.761857:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304044585 00000100:00000001:3.0:1788878432.761859:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878432.761860:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.761862:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878432.761866:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878432/real 1788878432] req@ffff94e511008000 x1875765290062848/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878432.761873:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878432.761874:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878432.761877:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878432.761879:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878432.761881:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.761884:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878432.761885:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878432.761891:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008000 x1875765290062848/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878432.761928:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.761932:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878432.761935:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.761938:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878432.761940:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.761943:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878432.761956:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878432.761960:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878432.761966:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878432.761969:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878432.761974:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290062848:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878432.761979:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878432.761981:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878432.761983:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878432.761987:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878432.761989:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878432.761992:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878432.761996:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290062976:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878432.762001:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878432.762004:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878432.762006:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878432.762008:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878432.762013:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.762021:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878432.762023:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286511616 : -117763423040000 : ffff94e511008e00) 00000100:00000001:3.0:1788878432.762026:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878432.762040:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878432.762044:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878432.762047:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878432.762048:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.762049:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878432.762052:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.762055:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304044042 00000100:00000001:3.0:1788878432.762057:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304044585 00000100:00000001:3.0:1788878432.762059:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304044042 00000100:00000001:3.0:1788878432.762061:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304044585 00000100:00000001:3.0:1788878432.762064:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878432.762066:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.762067:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878432.762071:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878432/real 1788878432] req@ffff94e511008e00 x1875765290062976/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878432.762078:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878432.762080:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878432.762082:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878432.762084:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878432.762086:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.762088:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878432.762089:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878432.762093:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008e00 x1875765290062976/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878432.762099:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.762101:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878432.762104:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.762106:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878432.762107:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.762109:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878432.762121:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878432.762124:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878432.762127:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878432.762129:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878432.762132:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290062976:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878432.762135:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878432.762136:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878432.762137:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878432.762139:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878432.762141:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878432.762143:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878432.762146:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290063104:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878432.762148:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878432.762150:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878432.762152:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878432.762153:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878432.762155:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.762159:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878432.762160:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286509824 : -117763423041792 : ffff94e511008700) 00000100:00000001:3.0:1788878432.762163:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878432.762171:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878432.762174:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878432.762176:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878432.762177:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.762178:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878432.762181:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.762183:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304044042 00000100:00000001:3.0:1788878432.762186:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304044585 00000100:00000001:3.0:1788878432.762188:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304044042 00000100:00000001:3.0:1788878432.762190:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304044585 00000100:00000001:3.0:1788878432.762193:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878432.762195:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.762196:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878432.762200:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878432/real 1788878432] req@ffff94e511008700 x1875765290063104/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878432.762207:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878432.762208:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878432.762210:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878432.762211:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878432.762212:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.762214:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878432.762215:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878432.762218:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008700 x1875765290063104/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878432.762229:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.762231:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878432.762233:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.762234:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878432.762235:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.762237:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878432.762246:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878432.762248:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878432.762250:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878432.762252:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878432.762254:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290063104:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878432.762257:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.762259:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878432.762260:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878432.762262:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878432.762264:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878432.762265:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878432.762266:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878432.762268:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878432.762269:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878432.762271:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878432.762272:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.762274:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878432.762275:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878432.762276:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878432.762277:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878432.762279:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878432.762279:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878432.762281:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878432.762282:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878432.762283:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878432.762284:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.762285:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878432.762286:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878432.762287:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878432.762289:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878432.762290:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878432.762291:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878432.762292:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878432.762293:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878432.762294:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878432.762295:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.762297:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878432.762300:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878432.762301:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878432.762302:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878432.762304:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878437.625125:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878437.625132:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878437.625137:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788878437.881458:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878437.881466:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878437.881473:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878437.881478:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878437.881483:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878437.881504:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878437.881510:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878437.881518:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878437.881528:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290063232:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878437.881538:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878437.881544:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878437.881549:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878437.881554:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878437.881563:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.881600:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878437.881629:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286523264 : -117763423028352 : ffff94e51100bb80) 00000100:00000001:3.0:1788878437.881638:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878437.881799:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878437.881842:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878437.881849:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878437.881855:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.881860:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878437.881870:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.881881:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304049705 00000100:00000001:3.0:1788878437.881919:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878437.881985:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304049706 00000100:00000001:3.0:1788878437.882025:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878437.882053:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878437.882060:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.882066:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878437.882080:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878437/real 1788878437] req@ffff94e51100bb80 x1875765290063232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878437.882105:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878437.882108:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878437.882115:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878437.882120:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878437.882122:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882126:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878437.882129:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878437.882135:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100bb80 x1875765290063232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878437.882163:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882167:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878437.882171:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882173:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878437.882174:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882177:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878437.882179:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878437.882183:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878437.882188:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878437.882191:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878437.882195:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290063232:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878437.882200:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878437.882203:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878437.882205:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878437.882208:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878437.882210:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878437.882229:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878437.882232:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290063360:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878437.882235:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878437.882237:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878437.882239:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878437.882241:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878437.882244:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.882252:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878437.882253:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:3.0:1788878437.882259:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878437.882273:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878437.882278:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878437.882280:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878437.882282:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.882283:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878437.882286:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.882290:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304049164 00000100:00000001:3.0:1788878437.882293:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304049705 00000100:00000001:3.0:1788878437.882296:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304049164 00000100:00000001:3.0:1788878437.882298:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304049706 00000100:00000001:3.0:1788878437.882300:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878437.882302:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.882303:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878437.882307:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878437/real 1788878437] req@ffff94e51100b800 x1875765290063360/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878437.882314:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878437.882315:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878437.882316:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878437.882318:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878437.882320:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882322:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878437.882323:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878437.882326:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290063360/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878437.882332:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882334:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878437.882336:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882338:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878437.882339:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882340:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878437.882358:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878437.882360:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878437.882363:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878437.882364:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878437.882367:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290063360:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878437.882370:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882373:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878437.882374:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878437.882377:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878437.882381:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878437.882383:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878437.882385:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878437.882390:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878437.882392:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878437.882395:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878437.882396:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882398:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878437.882399:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878437.882400:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878437.882402:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878437.882403:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878437.882404:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878437.882406:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878437.882407:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878437.882409:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878437.882410:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882412:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882420:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878437.882423:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.882425:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878437.882426:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878437.882427:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878437.882429:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878437.882430:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878437.882432:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878437.882433:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878437.882435:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878437.882437:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290063488:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878437.882440:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878437.882442:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878437.882443:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878437.882445:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878437.882447:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.882452:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878437.882454:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:3.0:1788878437.882457:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878437.882466:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878437.882470:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878437.882472:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878437.882473:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.882475:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878437.882477:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.882480:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304049162 00000100:00000001:3.0:1788878437.882483:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304049705 00000100:00000001:3.0:1788878437.882485:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304049162 00000100:00000001:3.0:1788878437.882487:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304049706 00000100:00000001:3.0:1788878437.882489:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878437.882490:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.882492:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878437.882496:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878437/real 1788878437] req@ffff94e51100b480 x1875765290063488/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878437.882503:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878437.882505:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878437.882506:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878437.882508:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878437.882509:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882511:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878437.882512:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878437.882516:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290063488/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878437.882521:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882523:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878437.882525:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882526:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878437.882527:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882529:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878437.882543:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878437.882545:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878437.882547:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878437.882549:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878437.882552:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290063488:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878437.882555:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878437.882556:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878437.882557:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878437.882559:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878437.882561:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878437.882562:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878437.882565:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290063616:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878437.882568:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878437.882569:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878437.882571:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878437.882573:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878437.882575:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.882578:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878437.882579:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:3.0:1788878437.882583:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878437.882590:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878437.882593:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878437.882595:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878437.882597:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.882600:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878437.882619:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.882625:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304049162 00000100:00000001:3.0:1788878437.882629:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304049705 00000100:00000001:3.0:1788878437.882633:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304049162 00000100:00000001:3.0:1788878437.882636:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304049706 00000100:00000001:3.0:1788878437.882640:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878437.882643:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.882647:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878437.882653:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878437/real 1788878437] req@ffff94e51100aa00 x1875765290063616/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878437.882668:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878437.882671:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878437.882676:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878437.882678:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878437.882681:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882684:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878437.882686:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878437.882694:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290063616/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878437.882707:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882711:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878437.882714:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882716:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878437.882717:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882719:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878437.882736:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878437.882739:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878437.882743:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878437.882746:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878437.882749:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290063616:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878437.882753:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882755:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878437.882779:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878437.882783:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878437.882786:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878437.882787:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878437.882789:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878437.882809:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878437.882811:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878437.882814:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878437.882815:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882818:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878437.882819:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878437.882821:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878437.882823:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878437.882824:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878437.882825:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878437.882827:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878437.882828:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878437.882830:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878437.882832:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882834:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878437.882839:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878437.882841:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878437.882843:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878437.882845:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878441.721074:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878441.721078:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878441.721083:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000020:00000001:3.0:1788878445.817037:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878445.817041:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878445.817045:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878448.121048:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878448.121061:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878448.121064:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878448.121069:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878448.121074:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878448.121083:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9386 00000100:00080000:3.0:1788878448.121097:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878448.121100:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878448.121102:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878448.121113:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878448.121116:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878448.121120:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878448.121122:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878448.121124:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878448.121125:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878448.121130:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878448.121132:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878448.121135:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878448.121140:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878448.121143:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121145:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878448.121148:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878448.121175:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878448.121177:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121181:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878448.121184:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878448.121190:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121206:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878448.121217:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290064256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878448.121229:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878448.121235:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121255:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878448.121259:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878448.121262:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878448.121264:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878448.121265:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878448.121269:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878448.121273:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9386 00000100:00080000:3.0:1788878448.121279:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878448.121283:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878448.121285:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878448.121291:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878448.121293:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878448.121297:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878448.121302:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878448.121309:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878448.121319:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878448.121331:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878448.121344:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878448.121352:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878448.121362:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878448.121369:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121373:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878448.121383:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878448.121392:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121397:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878448.121404:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878448.121408:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121411:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878448.121413:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878448.121424:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121434:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878448.121443:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290064384/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878448.121466:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878448.121469:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121475:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878448.121478:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878448.121482:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878448.121483:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878448.121485:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878448.121491:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878448.121494:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9386 00000100:00080000:3.0:1788878448.121499:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878448.121505:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878448.121509:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878448.121515:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878448.121518:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878448.121520:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878448.121526:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878448.121540:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878448.121544:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878448.121555:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878448.121565:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878448.121568:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878448.121574:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878448.121577:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121579:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878448.121582:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878448.121589:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121591:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878448.121596:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878448.121602:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121605:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878448.121607:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878448.121610:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121616:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878448.121624:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290064512/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878448.121641:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878448.121645:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121658:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878448.121682:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878448.121708:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878448.121709:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878448.121712:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878448.121720:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878448.121729:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9386 00000100:00080000:3.0:1788878448.121739:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878448.121751:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878448.121753:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878448.121766:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878448.121770:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878448.121775:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878448.121783:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878448.121794:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878448.121797:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878448.121808:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878448.121813:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878448.121825:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878448.121832:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878448.121843:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121847:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878448.121854:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878448.121876:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121895:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878448.121966:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878448.121970:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.121981:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878448.121984:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878448.121991:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.122006:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878448.122022:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290064640/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878448.122038:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878448.122045:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878448.122051:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878449.913094:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878449.913099:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878449.913103:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878453.241065:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878453.241077:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878453.241080:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878453.241084:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878453.241089:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878453.241098:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9392 00000100:00080000:2.0:1788878453.241110:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878453.241114:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878453.241116:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878453.241128:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878453.241131:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878453.241136:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878453.241150:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878453.241153:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878453.241154:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878453.241161:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878453.241164:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878453.241168:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878453.241173:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878453.241177:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241180:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878453.241183:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878453.241200:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878453.241201:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241215:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878453.241218:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878453.241224:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241239:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878453.241249:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290064768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878453.241260:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878453.241265:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241282:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878453.241285:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878453.241288:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878453.241289:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878453.241290:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878453.241293:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878453.241295:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9392 00000100:00080000:2.0:1788878453.241298:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878453.241301:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878453.241302:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878453.241306:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878453.241307:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878453.241309:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878453.241311:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878453.241324:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878453.241327:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878453.241333:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878453.241336:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878453.241339:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878453.241347:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878453.241351:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241353:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878453.241360:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878453.241365:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241369:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878453.241373:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878453.241376:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241378:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878453.241380:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878453.241382:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241395:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878453.241404:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290064896/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878453.241415:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878453.241418:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241420:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878453.241422:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878453.241425:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878453.241426:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878453.241427:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878453.241429:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878453.241431:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9392 00000100:00080000:2.0:1788878453.241433:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878453.241435:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878453.241440:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878453.241446:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878453.241449:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878453.241453:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878453.241455:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878453.241457:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878453.241458:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878453.241460:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878453.241462:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878453.241464:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878453.241466:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878453.241469:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241471:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878453.241474:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878453.241476:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241478:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878453.241483:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878453.241485:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241487:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878453.241489:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878453.241491:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241498:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878453.241509:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290065024/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878453.241526:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878453.241527:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241530:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878453.241533:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878453.241537:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878453.241542:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878453.241545:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878453.241551:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878453.241555:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9392 00000100:00080000:2.0:1788878453.241569:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878453.241572:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878453.241574:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878453.241590:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878453.241591:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878453.241593:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878453.241596:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878453.241602:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878453.241603:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878453.241608:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878453.241612:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878453.241618:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878453.241622:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878453.241627:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241631:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878453.241634:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878453.241640:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241645:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878453.241663:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878453.241666:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241668:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878453.241670:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878453.241673:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241683:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878453.241695:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290065152/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878453.241722:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878453.241727:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878453.241733:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878453.242259:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878453.242261:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290065152:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878453.242264:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878453.242266:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878453.242266:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878453.242268:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878453.242269:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878453.242270:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878453.242271:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878453.242282:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878453.242283:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878453.242284:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878453.242285:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878453.242286:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878453.242287:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878453.242288:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878453.242290:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878453.242290:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878453.242291:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878453.242292:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878453.242293:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878453.242294:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878453.242295:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878453.242296:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878453.242297:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878453.242298:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878453.242298:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878453.242299:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878453.242300:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878453.242301:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878453.242302:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878453.242303:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878453.242303:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878453.242305:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878453.242307:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878453.242308:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878453.242309:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878453.242310:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878454.009073:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878454.009086:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878454.009090:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878454.009096:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878454.009102:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878454.009105:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878454.009108:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878454.009111:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878454.009116:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878454.009118:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878454.009213:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878454.009218:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878454.009222:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878455.032995:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878455.033002:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878455.033008:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878455.033010:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878455.033040:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878455.033043:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878455.033046:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878456.057105:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878456.057113:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878456.057130:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878456.057132:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878456.057162:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878456.057165:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878456.057169:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878457.080988:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878457.080996:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878457.081001:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878457.081002:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878457.081025:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878457.081027:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878457.081029:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878458.105073:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878458.105083:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878458.105090:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878458.105091:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878458.105170:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878458.105175:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878458.105179:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878458.361108:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878458.361123:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878458.361126:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878458.361131:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878458.361139:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878458.361149:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9397 00000100:00080000:2.0:1788878458.361178:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878458.361181:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878458.361184:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878458.361194:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878458.361204:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878458.361209:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878458.361212:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878458.361215:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878458.361216:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878458.361221:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878458.361223:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878458.361226:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878458.361230:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878458.361233:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361235:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878458.361237:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878458.361252:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878458.361254:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361257:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878458.361259:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878458.361265:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361278:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878458.361287:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290065280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878458.361296:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878458.361300:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361317:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878458.361319:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878458.361322:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878458.361323:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878458.361325:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878458.361327:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878458.361329:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9397 00000100:00080000:2.0:1788878458.361332:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878458.361333:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878458.361334:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878458.361337:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878458.361338:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878458.361339:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878458.361341:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878458.361343:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878458.361344:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878458.361346:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878458.361348:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878458.361349:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878458.361355:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878458.361357:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361358:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878458.361362:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878458.361364:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361366:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.361367:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788878458.361369:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878458.361372:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361374:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878458.361375:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788878458.361375:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878458.361377:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.361381:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878458.361382:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.361387:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1788878458.361388:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290065408/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878458.361392:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361398:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1788878458.361400:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788878458.361401:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361405:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878458.361407:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788878458.361408:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1788878458.361414:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878458.361415:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878458.361418:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878458.361420:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878458.361423:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00100000:0.0:1788878458.361426:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290065280:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1788878458.361427:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9397 00000100:00080000:2.0:1788878458.361431:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878458.361435:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878458.361436:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878458.361437:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1788878458.361439:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878458.361441:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1788878458.361443:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:2.0:1788878458.361444:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878458.361448:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 02000000:00000001:0.0:1788878458.361449:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361450:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878458.361452:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:0.0:1788878458.361456:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1788878458.361457:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878458.361460:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878458.361461:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878458.361463:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878458.361464:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878458.361465:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361471:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878458.361472:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878458.361474:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361475:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878458.361496:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878458.361497:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361499:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878458.361500:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878458.361503:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.361510:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878458.361511:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.361519:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00100000:2.0:1788878458.361522:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290065536/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878458.361530:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878458.361544:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878458.361548:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361551:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878458.361554:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878458.361558:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878458.361560:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878458.361562:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878458.361565:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878458.361568:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9397 00000100:00080000:2.0:1788878458.361573:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878458.361576:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878458.361578:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878458.361582:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878458.361583:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878458.361586:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878458.361588:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878458.361590:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878458.361592:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878458.361599:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:0.0:1788878458.361604:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878458.361604:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878458.361610:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878458.361615:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878458.361659:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361662:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878458.361665:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878458.361672:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878458.361680:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00000001:2.0:1788878458.361681:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.361687:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361688:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.361696:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:2.0:1788878458.361700:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878458.361720:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.361723:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361723:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878458.361727:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878458.361731:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304070185 02000000:00000001:2.0:1788878458.361738:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361745:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878458.361749:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290065664/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878458.361766:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878458.361769:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878458.361769:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878458.361774:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.361786:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304070185 00000100:00000001:0.0:1788878458.361812:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878458.361818:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878458.361822:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.361826:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878458.361835:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878458/real 1788878458] req@ffff94e5045e4700 x1875765290065280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878458.361850:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878458.361852:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878458.361858:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878458.361861:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878458.361863:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.361866:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878458.361869:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878458.361892:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290065280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878458.361905:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.361909:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878458.361914:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.361917:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878458.361919:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.361922:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878458.361925:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878458.361929:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878458.361987:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878458.361990:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878458.361995:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290065280:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878458.362001:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362003:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878458.362005:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878458.362008:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878458.362013:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878458.362015:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878458.362018:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878458.362023:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878458.362025:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878458.362029:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878458.362030:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362033:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362043:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878458.362045:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.362047:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878458.362049:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878458.362051:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878458.362053:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878458.362054:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878458.362057:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878458.362059:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878458.362061:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878458.362065:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290065408:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878458.362069:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878458.362071:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878458.362073:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878458.362075:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878458.362078:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.362085:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878458.362087:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00000001:0.0:1788878458.362091:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878458.362107:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878458.362112:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878458.362114:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878458.362116:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.362118:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878458.362120:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.362125:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304069644 00000100:00000001:0.0:1788878458.362128:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304070185 00000100:00000001:0.0:1788878458.362130:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304069644 00000100:00000001:0.0:1788878458.362132:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304070185 00000100:00000001:0.0:1788878458.362135:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878458.362138:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.362139:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878458.362144:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878458/real 1788878458] req@ffff94e5045e7480 x1875765290065408/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878458.362154:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878458.362155:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878458.362158:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878458.362160:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878458.362162:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362164:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878458.362165:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878458.362170:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290065408/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878458.362192:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362194:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878458.362197:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362199:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878458.362200:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362202:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878458.362221:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878458.362224:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878458.362228:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878458.362230:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878458.362233:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290065408:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878458.362238:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878458.362239:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878458.362241:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878458.362243:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878458.362245:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878458.362247:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878458.362251:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290065536:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878458.362254:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878458.362256:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878458.362258:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878458.362260:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878458.362263:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.362269:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878458.362271:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00000001:0.0:1788878458.362274:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878458.362288:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878458.362292:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878458.362295:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878458.362296:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.362299:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878458.362302:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.362305:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304069642 00000100:00000001:0.0:1788878458.362308:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304070185 00000100:00000001:0.0:1788878458.362310:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304069642 00000100:00000001:0.0:1788878458.362312:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304070185 00000100:00000001:0.0:1788878458.362315:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878458.362317:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.362319:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878458.362323:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878458/real 1788878458] req@ffff94e5045e7b80 x1875765290065536/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878458.362333:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878458.362335:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878458.362337:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878458.362339:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878458.362340:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362342:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878458.362344:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878458.362348:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290065536/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878458.362356:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362359:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878458.362362:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362363:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878458.362364:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362366:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878458.362382:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878458.362385:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878458.362389:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878458.362392:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878458.362396:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290065536:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878458.362400:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878458.362401:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878458.362403:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878458.362405:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878458.362407:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878458.362410:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878458.362413:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290065664:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878458.362416:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878458.362418:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878458.362420:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878458.362421:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878458.362425:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.362430:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878458.362432:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:0.0:1788878458.362436:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878458.362446:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878458.362451:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878458.362454:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878458.362455:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.362458:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878458.362460:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.362464:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304069642 00000100:00000001:0.0:1788878458.362467:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304070185 00000100:00000001:0.0:1788878458.362469:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304069642 00000100:00000001:0.0:1788878458.362472:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304070185 00000100:00000001:0.0:1788878458.362474:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878458.362477:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.362479:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878458.362483:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878458/real 1788878458] req@ffff94e5045e4e00 x1875765290065664/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878458.362500:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878458.362502:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878458.362505:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878458.362507:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878458.362508:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362510:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878458.362512:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878458.362516:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290065664/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878458.362524:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362527:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878458.362529:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362531:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878458.362532:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362535:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878458.362550:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878458.362554:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878458.362557:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878458.362560:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878458.362563:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290065664:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878458.362568:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362570:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878458.362571:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878458.362573:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878458.362576:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878458.362578:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878458.362579:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878458.362581:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878458.362583:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878458.362585:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878458.362587:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362589:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878458.362591:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878458.362592:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878458.362594:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878458.362595:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878458.362596:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878458.362598:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878458.362599:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878458.362601:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878458.362602:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362604:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878458.362605:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878458.362606:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878458.362608:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878458.362609:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878458.362610:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878458.362612:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878458.362614:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878458.362616:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878458.362630:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362633:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878458.362636:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878458.362638:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878458.362640:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878458.362641:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878459.128990:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878459.128998:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878459.129003:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878459.129004:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878459.129025:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878459.129027:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878459.129030:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878460.153010:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878460.153017:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878460.153021:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878460.153026:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878460.153029:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878460.153033:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878460.153037:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878460.153039:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878460.153042:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878460.153045:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878460.153081:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878460.153083:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878460.153086:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878461.177013:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878461.177024:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878461.177031:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878461.177032:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878461.177061:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878461.177065:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878461.177068:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878462.200975:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878462.200981:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878462.200985:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878462.200986:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878462.201054:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878462.201057:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878462.201060:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878463.224979:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878463.224987:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878463.224989:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878463.224993:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878463.224997:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878463.224999:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878463.225001:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878463.225003:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878463.225007:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878463.225009:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878463.225029:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878463.225031:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878463.225034:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878463.480982:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878463.480991:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878463.480993:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878463.480996:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878463.481000:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878463.481006:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9402 00000100:00080000:2.0:1788878463.481016:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878463.481018:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878463.481020:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878463.481025:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878463.481028:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878463.481031:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878463.481033:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878463.481035:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878463.481036:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878463.481040:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878463.481041:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878463.481044:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878463.481046:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878463.481048:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481049:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878463.481051:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878463.481060:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878463.481062:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481064:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878463.481067:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878463.481070:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481079:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878463.481086:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290065792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878463.481095:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878463.481098:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481111:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878463.481115:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878463.481135:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878463.481138:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878463.481141:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878463.481145:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878463.481150:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9402 00000100:00080000:2.0:1788878463.481155:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878463.481158:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878463.481161:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878463.481165:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878463.481167:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878463.481170:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878463.481173:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878463.481177:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878463.481179:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878463.481184:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878463.481188:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878463.481188:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878463.481191:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878463.481194:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878463.481197:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788878463.481198:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878463.481201:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788878463.481201:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481204:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481204:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:0.0:1788878463.481210:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:2.0:1788878463.481210:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788878463.481213:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1788878463.481215:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878463.481217:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481218:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878463.481224:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290065792:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878463.481224:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878463.481226:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481229:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878463.481231:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878463.481234:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481235:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878463.481240:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788878463.481241:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878463.481243:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878463.481247:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1788878463.481249:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4000 x1875765290065920/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788878463.481253:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481278:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878463.481282:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481286:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481290:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788878463.481291:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878463.481294:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074559872 : -117763634991744 : ffff94e5045e6d80) 00000100:00080000:2.0:1788878463.481297:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878463.481299:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878463.481300:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788878463.481302:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878463.481307:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878463.481310:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9402 00000100:00080000:2.0:1788878463.481315:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878463.481319:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878463.481321:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878463.481325:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878463.481327:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878463.481330:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878463.481334:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878463.481337:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878463.481339:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878463.481354:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:0.0:1788878463.481356:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878463.481358:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878463.481362:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878463.481366:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878463.481369:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481372:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878463.481373:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00100000:2.0:1788878463.481375:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878463.481378:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00000001:2.0:1788878463.481379:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481382:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481382:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481386:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:2.0:1788878463.481390:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878463.481393:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878463.481393:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481396:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878463.481399:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304075305 02000000:00000001:2.0:1788878463.481399:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878463.481402:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481409:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481412:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00100000:2.0:1788878463.481417:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290066048/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878463.481437:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304075305 00000100:00000001:0.0:1788878463.481443:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878463.481444:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878463.481447:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878463.481447:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481450:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481450:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481453:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1788878463.481453:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878463.481457:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1788878463.481459:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878463/real 1788878463] req@ffff94e5045e6d80 x1875765290065792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878463.481459:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878463.481461:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878463.481465:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878463.481468:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9402 00000100:00080000:2.0:1788878463.481472:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000400:00000001:0.0:1788878463.481473:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878463.481475:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878463.481475:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878463.481477:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878463.481480:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000020:00000001:2.0:1788878463.481480:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878463.481482:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000020:00000001:2.0:1788878463.481482:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878463.481485:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788878463.481485:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878463.481488:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1788878463.481488:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878463.481491:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878463.481491:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878463.481493:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:0.0:1788878463.481496:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6d80 x1875765290065792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878463.481497:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878463.481501:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878463.481503:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878463.481506:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878463.481506:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878463.481508:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481527:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878463.481529:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878463.481531:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481533:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481536:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788878463.481537:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878463.481538:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481539:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878463.481540:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878463.481541:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1788878463.481542:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878463.481543:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878463.481544:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481545:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878463.481548:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878463.481549:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878463.481551:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788878463.481554:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290066176/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878463.481555:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878463.481558:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788878463.481562:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1788878463.481563:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290065792:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878463.481564:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878463.481566:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481573:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.481576:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878463.481578:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878463.481580:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878463.481597:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878463.481599:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878463.481601:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878463.481606:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878463.481608:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878463.481611:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878463.481613:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.481616:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.481647:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878463.481649:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481650:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878463.481651:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878463.481652:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878463.481654:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878463.481655:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878463.481656:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878463.481658:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878463.481659:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878463.481662:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290065920:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878463.481665:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878463.481666:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878463.481668:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878463.481669:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878463.481671:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481676:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878463.481677:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074548224 : -117763635003392 : ffff94e5045e4000) 00000100:00000001:0.0:1788878463.481681:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878463.481707:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878463.481710:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878463.481712:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878463.481713:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481714:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878463.481716:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481719:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304074763 00000100:00000001:0.0:1788878463.481721:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304075305 00000100:00000001:0.0:1788878463.481722:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304074763 00000100:00000001:0.0:1788878463.481724:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304075305 00000100:00000001:0.0:1788878463.481726:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878463.481727:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481728:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878463.481731:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878463/real 1788878463] req@ffff94e5045e4000 x1875765290065920/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878463.481736:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878463.481737:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878463.481739:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878463.481740:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878463.481741:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.481742:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878463.481743:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878463.481746:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4000 x1875765290065920/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878463.481750:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.481752:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878463.481754:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.481755:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878463.481756:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.481757:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878463.481768:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878463.481781:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878463.481783:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878463.481784:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878463.481786:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290065920:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878463.481794:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878463.481795:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878463.481796:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878463.481797:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878463.481798:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878463.481799:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878463.481801:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290066048:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878463.481804:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878463.481805:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878463.481806:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878463.481807:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878463.481809:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481813:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878463.481814:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:0.0:1788878463.481816:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878463.481823:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878463.481825:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878463.481827:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878463.481828:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481829:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878463.481831:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481833:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304074761 00000100:00000001:0.0:1788878463.481835:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304075305 00000100:00000001:0.0:1788878463.481837:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304074761 00000100:00000001:0.0:1788878463.481838:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304075305 00000100:00000001:0.0:1788878463.481840:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878463.481841:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481843:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878463.481845:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878463/real 1788878463] req@ffff94e5045e4e00 x1875765290066048/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878463.481850:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878463.481850:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878463.481852:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878463.481853:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878463.481860:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.481862:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878463.481863:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878463.481865:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290066048/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878463.481870:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.481871:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878463.481873:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.481874:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878463.481875:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.481876:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878463.481885:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878463.481886:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878463.481888:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878463.481889:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878463.481891:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290066048:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878463.481894:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878463.481895:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878463.481896:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878463.481897:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878463.481898:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878463.481900:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878463.481919:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290066176:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878463.481922:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878463.481923:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878463.481924:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878463.481926:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878463.481928:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481931:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878463.481932:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00000001:0.0:1788878463.481935:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878463.481942:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878463.481945:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878463.481946:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878463.481947:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481948:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878463.481950:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481952:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304074762 00000100:00000001:0.0:1788878463.481954:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304075305 00000100:00000001:0.0:1788878463.481956:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304074762 00000100:00000001:0.0:1788878463.481957:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304075305 00000100:00000001:0.0:1788878463.481959:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878463.481960:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.481961:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878463.481964:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878463/real 1788878463] req@ffff94e5045e7b80 x1875765290066176/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878463.481969:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878463.481969:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878463.481971:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878463.481972:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878463.481973:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.481975:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878463.481976:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878463.481978:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290066176/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878463.481983:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.481984:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878463.481985:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.481986:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878463.481987:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.481988:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878463.481997:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878463.481999:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878463.482001:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878463.482002:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878463.482004:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290066176:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878463.482006:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.482007:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878463.482008:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878463.482009:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878463.482010:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878463.482012:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878463.482012:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878463.482014:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878463.482015:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878463.482016:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878463.482017:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.482018:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878463.482019:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878463.482020:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878463.482021:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878463.482022:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878463.482023:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878463.482024:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878463.482024:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878463.482026:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878463.482027:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.482028:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878463.482029:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878463.482030:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878463.482031:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878463.482031:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878463.482032:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878463.482033:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878463.482034:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878463.482035:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878463.482036:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.482037:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878463.482039:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878463.482040:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878463.482041:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878463.482042:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878464.248992:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878464.249000:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878464.249007:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878464.249008:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878464.249029:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878464.249032:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878464.249035:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878465.272986:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878465.272993:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878465.272998:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878465.272999:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878465.273019:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878465.273022:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878465.273024:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878466.297041:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878466.297052:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878466.297061:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878466.297064:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878466.297190:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878466.297198:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878466.297205:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878467.321004:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878467.321011:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878467.321017:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878467.321019:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878467.321042:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878467.321046:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878467.321050:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878468.344980:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878468.344988:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878468.344994:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878468.344995:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878468.345016:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878468.345018:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878468.345021:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878468.601038:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878468.601072:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878468.601076:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878468.601082:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878468.601090:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878468.601102:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9407 00000100:00080000:0.0:1788878468.601142:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878468.601147:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878468.601151:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878468.601163:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878468.601168:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878468.601174:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878468.601179:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878468.601182:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878468.601184:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878468.601191:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878468.601194:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878468.601198:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878468.601203:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878468.601206:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878468.601209:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878468.601212:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878468.601225:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878468.601227:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878468.601232:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878468.601235:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878468.601240:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878468.601254:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878468.601265:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290066304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878468.601278:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878468.601284:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878468.601319:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878468.601322:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878468.601325:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878468.601327:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878468.601328:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878468.601330:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878468.601332:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9407 00000100:00080000:0.0:1788878468.601335:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878468.601338:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878468.601339:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878468.601342:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878468.601344:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878468.601346:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788878468.601348:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878468.601350:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878468.601352:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878468.601355:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878468.601374:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.601374:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788878468.601382:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788878468.601384:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788878468.601390:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878468.601391:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878468.601398:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788878468.601399:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.601403:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878468.601405:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878468.601413:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788878468.601414:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:0.0:1788878468.601423:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878468.601423:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878468.601431:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878468.601433:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878468.601439:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878468.601445:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878468.601446:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290066304:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878468.601448:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878468.601453:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788878468.601459:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878468.601461:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878468.601467:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788878468.601472:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878468.601473:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878468.601483:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1788878468.601489:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290066432/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788878468.601490:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878468.601507:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878468.601511:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878468.601515:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878468.601518:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878468.601521:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878468.601523:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878468.601525:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878468.601529:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788878468.601531:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1788878468.601533:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9407 00000100:00000001:3.0:1788878468.601536:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:3.0:1788878468.601566:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788878468.601567:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878468.601576:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878468.601579:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878468.601584:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878468.601586:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878468.601589:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878468.601592:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878468.601595:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878468.601597:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878468.601602:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878468.601605:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878468.601608:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878468.601614:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878468.601616:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878468.601618:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878468.601622:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878468.601625:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878468.601631:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788878468.601635:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:0.0:1788878468.601639:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878468.601644:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878468.601647:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878468.601649:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878468.601653:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.601658:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878468.601663:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.601668:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878468.601674:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.601678:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:0.0:1788878468.601681:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290066560/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878468.601687:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.601698:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304080425 00000100:00000001:0.0:1788878468.601711:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878468.601716:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878468.601718:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878468.601730:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.601730:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304080425 00000100:00080000:0.0:1788878468.601740:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788878468.601741:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878468.601750:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1788878468.601752:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878468.601756:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.601761:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788878468.601792:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1788878468.601794:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878468/real 1788878468] req@ffff94e50897ad80 x1875765290066304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878468.601797:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878468.601808:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878468.601818:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9407 00000100:00080000:0.0:1788878468.601829:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878468.601831:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878468.601833:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878468.601838:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878468.601840:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000001:3.0:1788878468.601842:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:0.0:1788878468.601843:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000400:00000001:3.0:1788878468.601847:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:0.0:1788878468.601852:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878468.601857:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878468.601859:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878468.601864:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878468.601866:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878468.601871:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788878468.601876:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878468.601880:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878468.601882:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.601886:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788878468.601888:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878468.601896:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788878468.601899:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290066304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788878468.601944:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878468.601948:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878468.601951:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878468.601954:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878468.601957:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878468.601963:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878468.601964:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878468.601967:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878468.601968:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878468.601971:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878468.601993:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878468.601999:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290066688/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878468.602030:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878468.602030:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878468.602034:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.602054:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878468.602056:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.602071:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602078:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878468.602082:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602087:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878468.602092:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878468.602099:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878468.602142:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878468.602148:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878468.602156:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290066304:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878468.602163:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602166:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878468.602168:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878468.602171:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878468.602177:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878468.602180:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878468.602196:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878468.602203:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878468.602206:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878468.602213:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878468.602216:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602220:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602230:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878468.602234:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.602238:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878468.602241:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878468.602244:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878468.602247:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878468.602250:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878468.602254:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878468.602259:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878468.602263:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878468.602286:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290066432:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878468.602290:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878468.602292:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878468.602293:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878468.602295:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878468.602297:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.602304:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878468.602305:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:3.0:1788878468.602309:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878468.602323:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878468.602327:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878468.602329:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878468.602330:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.602331:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878468.602352:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.602357:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304079884 00000100:00000001:3.0:1788878468.602361:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304080425 00000100:00000001:3.0:1788878468.602364:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304079884 00000100:00000001:3.0:1788878468.602367:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304080425 00000100:00000001:3.0:1788878468.602369:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878468.602372:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.602374:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878468.602380:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878468/real 1788878468] req@ffff94e50897bb80 x1875765290066432/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878468.602390:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878468.602392:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878468.602395:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878468.602397:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878468.602399:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602401:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878468.602419:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878468.602425:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290066432/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878468.602435:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602438:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878468.602442:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602444:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878468.602445:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602447:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878468.602461:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878468.602464:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878468.602468:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878468.602470:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878468.602474:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290066432:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878468.602479:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878468.602481:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878468.602483:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878468.602486:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878468.602488:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878468.602491:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878468.602494:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290066560:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878468.602499:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878468.602501:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878468.602503:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878468.602505:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878468.602508:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.602515:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878468.602517:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:3.0:1788878468.602521:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878468.602536:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878468.602541:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878468.602543:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878468.602586:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.602590:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878468.602593:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.602598:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304079882 00000100:00000001:3.0:1788878468.602601:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304080425 00000100:00000001:3.0:1788878468.602604:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304079882 00000100:00000001:3.0:1788878468.602607:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304080425 00000100:00000001:3.0:1788878468.602609:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878468.602612:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.602614:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878468.602619:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878468/real 1788878468] req@ffff94e508978e00 x1875765290066560/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878468.602630:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878468.602649:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878468.602654:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878468.602658:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878468.602660:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602665:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878468.602667:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878468.602675:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290066560/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878468.602684:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602687:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878468.602689:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602691:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878468.602693:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602695:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878468.602706:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878468.602709:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878468.602712:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878468.602715:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878468.602719:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290066560:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878468.602724:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878468.602726:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878468.602728:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878468.602730:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878468.602732:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878468.602735:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878468.602738:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290066688:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878468.602742:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878468.602744:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878468.602746:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878468.602748:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878468.602752:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.602759:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878468.602761:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:3.0:1788878468.602765:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878468.602810:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878468.602817:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878468.602820:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878468.602822:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.602824:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878468.602827:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.602832:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304079882 00000100:00000001:3.0:1788878468.602835:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304080425 00000100:00000001:3.0:1788878468.602838:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304079882 00000100:00000001:3.0:1788878468.602841:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304080425 00000100:00000001:3.0:1788878468.602844:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878468.602846:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.602849:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878468.602853:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878468/real 1788878468] req@ffff94e508978a80 x1875765290066688/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878468.602872:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878468.602873:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878468.602876:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878468.602878:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878468.602880:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602882:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878468.602884:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878468.602888:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290066688/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878468.602896:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602899:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878468.602902:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602904:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878468.602906:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602908:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878468.602920:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878468.602923:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878468.602927:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878468.602929:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878468.602933:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290066688:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878468.602938:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.602941:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878468.602972:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878468.602975:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878468.602978:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878468.602980:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878468.602998:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878468.603000:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878468.603001:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878468.603003:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878468.603004:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.603006:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878468.603007:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878468.603008:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878468.603010:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878468.603011:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878468.603012:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878468.603014:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878468.603014:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878468.603016:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878468.603017:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.603018:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878468.603019:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878468.603020:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878468.603021:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878468.603022:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878468.603023:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878468.603024:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878468.603025:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878468.603026:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878468.603027:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.603028:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878468.603031:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878468.603032:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878468.603034:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878468.603035:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878469.369027:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878469.369036:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878469.369042:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878469.369044:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878469.369072:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878469.369075:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878469.369078:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878470.392958:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878470.392963:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878470.392968:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878470.392970:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878470.392972:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878470.392974:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878470.392976:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878470.392978:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878470.392980:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878470.392981:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878470.393027:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878470.393030:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878470.393033:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878471.417013:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878471.417022:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878471.417028:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878471.417030:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878471.417056:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878471.417059:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878471.417063:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878472.441016:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878472.441024:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878472.441031:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878472.441033:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878472.441058:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878472.441062:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878472.441065:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878473.465138:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878473.465148:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878473.465152:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878473.465156:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878473.465162:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878473.465165:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878473.465168:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878473.465171:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878473.465175:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878473.465177:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878473.465243:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878473.465264:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878473.465268:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878473.721148:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878473.721166:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878473.721171:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878473.721187:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878473.721195:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878473.721207:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9412 00000100:00080000:3.0:1788878473.721225:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878473.721229:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878473.721232:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878473.721245:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878473.721250:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878473.721256:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878473.721259:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878473.721262:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878473.721264:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878473.721270:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878473.721274:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878473.721278:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878473.721283:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878473.721299:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878473.721302:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878473.721306:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878473.721325:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878473.721327:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878473.721332:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878473.721336:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878473.721344:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878473.721365:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878473.721379:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290066816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878473.721394:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878473.721401:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878473.721433:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878473.721438:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878473.721443:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878473.721445:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878473.721447:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878473.721452:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878473.721458:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9412 00000100:00080000:3.0:1788878473.721462:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878473.721465:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878473.721470:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878473.721476:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878473.721477:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878473.721480:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878473.721483:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878473.721486:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878473.721489:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878473.721493:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1788878473.721496:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878473.721508:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878473.721508:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.721531:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788878473.721550:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878473.721555:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878473.721560:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878473.721561:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878473.721579:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878473.721582:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878473.721590:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878473.721590:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:0.0:1788878473.721602:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788878473.721606:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878473.721616:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290066816:192.168.203.129@tcp:250:kworker.0 00000008:00000001:3.0:1788878473.721620:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.721629:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788878473.721634:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878473.721639:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878473.721646:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878473.721657:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878473.721657:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878473.721671:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878473.721678:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878473.721680:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878473.721688:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878473.721694:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878473.721705:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.721707:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878473.721712:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:0.0:1788878473.721723:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1788878473.721723:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290066944/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878473.721758:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878473.721762:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878473.721770:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878473.721774:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878473.721779:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878473.721781:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878473.721784:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878473.721789:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878473.721795:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9412 00000100:00080000:3.0:1788878473.721804:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1788878473.721805:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878473.721813:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878473.721817:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878473.721828:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878473.721829:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:3.0:1788878473.721832:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878473.721837:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878473.721838:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1788878473.721847:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878473.721848:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878473.721871:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878473.721873:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878473.721885:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878473.721889:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878473.721898:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878473.721903:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304085545 00000100:00000001:0.0:1788878473.721921:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878473.721947:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304085546 00000100:00000001:0.0:1788878473.721989:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878473.721995:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878473.722000:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.722004:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788878473.722032:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878473.722033:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878473/real 1788878473] req@ffff94e50449c000 x1875765290066816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788878473.722046:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878473.722064:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:0.0:1788878473.722066:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878473.722070:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788878473.722080:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.722086:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878473.722094:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878473.722095:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878473.722104:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1788878473.722107:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878473.722110:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878473.722115:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1788878473.722122:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878473.722134:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878473.722135:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290066816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788878473.722154:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878473.722162:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878473.722169:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878473.722171:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878473.722174:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878473.722193:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878473.722204:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290067072/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878473.722222:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.722240:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878473.722243:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878473.722247:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878473.722251:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.722257:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878473.722261:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.722262:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.722283:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1788878473.722285:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878473.722288:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878473.722309:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878473.722325:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878473.722328:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878473.722330:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878473.722341:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878473.722349:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9412 00000100:00080000:0.0:1788878473.722354:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00080000:3.0:1788878473.722373:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878473.722375:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788878473.722390:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1788878473.722404:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290066816:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878473.722404:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878473.722429:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878473.722430:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788878473.722440:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878473.722441:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1788878473.722448:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878473.722449:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878473.722461:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:3.0:1788878473.722464:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878473.722476:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788878473.722478:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1788878473.722481:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878473.722484:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878473.722489:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00080000:3.0:1788878473.722491:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878473.722499:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878473.722502:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878473.722505:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878473.722506:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:3.0:1788878473.722551:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878473.722555:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878473.722562:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878473.722562:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.722586:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878473.722587:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878473.722597:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878473.722601:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878473.722606:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788878473.722612:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.722614:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878473.722619:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878473.722626:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878473.722628:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.722633:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878473.722638:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878473.722643:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878473.722646:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878473.722648:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878473.722650:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878473.722657:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878473.722663:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290066944:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878473.722670:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878473.722672:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878473.722676:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878473.722679:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878473.722683:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878473.722684:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878473.722700:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878473.722706:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878473.722706:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878473.722747:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290067200/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878473.722757:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878473.722770:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:3.0:1788878473.722770:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878473.722784:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.722787:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878473.722798:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788878473.722819:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878473.722827:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878473.722836:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878473.722839:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.722842:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878473.722846:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.722852:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304085004 00000100:00000001:0.0:1788878473.722857:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304085545 00000100:00000001:0.0:1788878473.722861:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304085004 00000100:00000001:0.0:1788878473.722867:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304085546 00000100:00000001:0.0:1788878473.722871:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878473.722874:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.722876:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878473.722883:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878473/real 1788878473] req@ffff94e50449d180 x1875765290066944/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878473.722893:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878473.722895:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878473.722898:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878473.722900:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878473.722902:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.722904:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878473.722907:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878473.722911:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290066944/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878473.722930:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.722933:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878473.722936:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.722986:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878473.722987:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.722989:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878473.723003:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878473.723007:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878473.723011:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878473.723014:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878473.723035:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290066944:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878473.723042:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878473.723045:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878473.723048:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878473.723053:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878473.723057:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878473.723062:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878473.723068:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290067072:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878473.723075:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878473.723079:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878473.723082:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878473.723086:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878473.723092:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.723105:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878473.723108:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:0.0:1788878473.723115:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878473.723140:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878473.723150:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878473.723155:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878473.723158:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.723162:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878473.723167:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.723174:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304085003 00000100:00000001:0.0:1788878473.723195:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304085545 00000100:00000001:0.0:1788878473.723200:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304085003 00000100:00000001:0.0:1788878473.723204:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304085546 00000100:00000001:0.0:1788878473.723208:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878473.723212:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.723215:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878473.723223:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878473/real 1788878473] req@ffff94e50449f800 x1875765290067072/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878473.723239:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878473.723241:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878473.723246:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878473.723249:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878473.723251:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.723254:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878473.723257:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878473.723264:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290067072/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878473.723274:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.723277:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878473.723280:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.723281:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878473.723283:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.723285:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878473.723297:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878473.723300:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878473.723303:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878473.723306:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878473.723311:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290067072:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878473.723315:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.723318:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878473.723320:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878473.723322:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878473.723325:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878473.723327:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878473.723329:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878473.723332:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878473.723334:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878473.723336:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878473.723338:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.723340:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878473.723341:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878473.723343:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878473.723345:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878473.723346:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878473.723347:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878473.723349:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878473.723350:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878473.723352:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878473.723353:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.723355:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.723359:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878473.723361:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.723364:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878473.723366:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878473.723367:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878473.723369:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878473.723371:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878473.723373:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878473.723375:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878473.723378:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878473.723381:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290067200:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878473.723385:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878473.723388:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878473.723390:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878473.723392:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878473.723396:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.723404:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878473.723405:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:0.0:1788878473.723410:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878473.723425:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878473.723430:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878473.723433:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878473.723434:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.723436:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878473.723439:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.723444:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304085003 00000100:00000001:0.0:1788878473.723447:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304085545 00000100:00000001:0.0:1788878473.723449:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304085003 00000100:00000001:0.0:1788878473.723452:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304085546 00000100:00000001:0.0:1788878473.723454:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878473.723457:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.723459:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878473.723463:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878473/real 1788878473] req@ffff94e50449ca80 x1875765290067200/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878473.723471:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878473.723473:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878473.723475:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878473.723477:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878473.723479:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.723481:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878473.723483:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878473.723487:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290067200/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878473.723495:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.723497:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878473.723500:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.723502:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878473.723504:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.723506:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878473.723541:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878473.723545:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878473.723550:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878473.723553:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878473.723557:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290067200:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878473.723562:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.723564:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878473.723565:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878473.723567:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878473.723570:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878473.723571:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878473.723573:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878473.723575:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878473.723576:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878473.723579:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878473.723580:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.723583:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878473.723586:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878473.723588:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878473.723590:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878473.723592:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878474.489019:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878474.489024:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878474.489028:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878474.489029:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878474.489236:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878474.489244:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878474.489267:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878475.513011:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878475.513018:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878475.513023:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878475.513024:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878475.513047:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878475.513049:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878475.513052:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878476.537009:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878476.537014:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878476.537019:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878476.537020:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878476.537037:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878476.537039:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878476.537041:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878477.561005:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878477.561013:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878477.561019:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878477.561021:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878477.561046:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878477.561049:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878477.561053:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878478.585059:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878478.585065:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878478.585069:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878478.585070:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878478.585117:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878478.585120:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878478.585123:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878478.840992:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878478.841002:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878478.841004:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878478.841008:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878478.841012:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878478.841018:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9417 00000100:00080000:3.0:1788878478.841028:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878478.841031:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878478.841033:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878478.841041:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878478.841044:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878478.841047:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878478.841049:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878478.841051:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878478.841052:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878478.841064:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878478.841066:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878478.841089:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878478.841092:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878478.841094:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878478.841096:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878478.841098:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878478.841107:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878478.841108:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878478.841111:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878478.841113:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878478.841117:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878478.841126:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878478.841133:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290067328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878478.841141:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878478.841145:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878478.841156:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878478.841169:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878478.841172:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878478.841173:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878478.841174:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878478.841176:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878478.841178:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9417 00000100:00080000:3.0:1788878478.841181:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878478.841183:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878478.841184:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878478.841186:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878478.841188:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878478.841189:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878478.841191:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878478.841193:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878478.841194:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878478.841197:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1788878478.841198:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878478.841202:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841203:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788878478.841203:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878478.841207:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878478.841209:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788878478.841210:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878478.841211:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878478.841214:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878478.841217:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878478.841217:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:0.0:1788878478.841222:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1788878478.841223:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788878478.841226:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788878478.841227:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878478.841242:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878478.841243:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290067328:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788878478.841245:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878478.841248:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841249:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788878478.841251:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878478.841253:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878478.841253:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878478.841255:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878478.841256:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878478.841259:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878478.841260:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878478.841265:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878478.841266:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290067456/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878478.841273:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878478.841275:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878478.841277:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878478.841279:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878478.841281:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788878478.841281:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878478.841283:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:3.0:1788878478.841284:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878478.841288:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878478.841289:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878478.841293:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878478.841296:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9417 00000100:00080000:3.0:1788878478.841298:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878478.841299:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878478.841300:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878478.841303:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878478.841303:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878478.841305:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878478.841306:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878478.841307:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878478.841308:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878478.841310:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878478.841312:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878478.841313:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878478.841315:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878478.841317:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878478.841319:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878478.841320:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788878478.841324:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00000001:3.0:1788878478.841324:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878478.841326:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878478.841331:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878478.841333:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878478.841334:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878478.841335:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788878478.841335:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878478.841336:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841341:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878478.841371:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841374:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878478.841377:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841378:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841384:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304090665 00000100:00100000:3.0:1788878478.841386:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290067584/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878478.841393:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878478.841394:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878478.841396:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878478.841400:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841403:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304090665 00000100:00080000:3.0:1788878478.841403:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878478.841426:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788878478.841430:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878478.841431:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878478.841433:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878478.841434:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878478.841436:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878478.841438:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788878478.841440:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878478.841444:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9417 00000100:00100000:0.0:1788878478.841448:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878478/real 1788878478] req@ffff94e50449f100 x1875765290067328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878478.841449:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878478.841452:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878478.841454:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878478.841458:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878478.841460:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878478.841463:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878478.841467:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878478.841470:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878478.841471:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:0.0:1788878478.841472:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1788878478.841496:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000400:00000001:0.0:1788878478.841499:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878478.841500:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878478.841503:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878478.841506:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878478.841511:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:3.0:1788878478.841511:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878478.841514:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878478.841515:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841517:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878478.841518:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878478.841521:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1788878478.841522:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878478.841525:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878478.841528:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878478.841529:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290067328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788878478.841532:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878478.841535:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878478.841538:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878478.841557:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878478.841558:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878478.841563:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878478.841564:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841567:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.841569:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878478.841570:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.841573:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878478.841573:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841575:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878478.841580:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878478.841587:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1788878478.841588:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290067712/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878478.841589:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878478.841605:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290067328:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878478.841635:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.841638:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878478.841639:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878478.841641:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878478.841644:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878478.841646:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:3.0:1788878478.841648:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000020:00000001:0.0:1788878478.841649:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788878478.841652:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878478.841654:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1788878478.841656:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878478.841657:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878478.841661:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878478.841662:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.841664:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.841670:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878478.841672:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841674:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878478.841675:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878478.841676:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878478.841677:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878478.841679:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878478.841696:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878478.841698:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878478.841700:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878478.841703:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290067456:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878478.841706:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878478.841708:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878478.841709:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878478.841711:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878478.841714:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841719:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878478.841720:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:0.0:1788878478.841724:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878478.841737:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878478.841741:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878478.841743:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878478.841744:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841746:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878478.841750:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841755:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304090123 00000100:00000001:0.0:1788878478.841758:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304090665 00000100:00000001:0.0:1788878478.841761:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304090123 00000100:00000001:0.0:1788878478.841764:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304090665 00000100:00000001:0.0:1788878478.841767:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878478.841785:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841787:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878478.841792:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878478/real 1788878478] req@ffff94e50449e300 x1875765290067456/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878478.841803:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878478.841804:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878478.841807:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878478.841809:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878478.841811:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.841813:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878478.841815:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878478.841820:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290067456/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878478.841828:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.841831:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878478.841850:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.841852:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878478.841853:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.841854:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878478.841867:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878478.841870:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878478.841874:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878478.841877:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878478.841880:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290067456:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878478.841887:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878478.841889:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878478.841891:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878478.841893:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878478.841895:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878478.841897:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878478.841900:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290067584:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878478.841921:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878478.841923:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878478.841925:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878478.841928:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878478.841932:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841938:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878478.841940:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:0.0:1788878478.841945:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878478.841964:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878478.841971:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878478.841975:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878478.841977:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841980:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878478.841985:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.841989:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304090122 00000100:00000001:0.0:1788878478.841993:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304090665 00000100:00000001:0.0:1788878478.842041:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304090122 00000100:00000001:0.0:1788878478.842044:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304090665 00000100:00000001:0.0:1788878478.842047:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878478.842050:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.842052:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878478.842058:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878478/real 1788878478] req@ffff94e50449ca80 x1875765290067584/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878478.842066:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878478.842068:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878478.842087:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878478.842088:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878478.842089:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.842091:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878478.842092:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878478.842094:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290067584/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878478.842099:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.842101:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878478.842102:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.842103:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878478.842104:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.842105:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878478.842114:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878478.842116:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878478.842118:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878478.842119:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878478.842121:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290067584:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878478.842123:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878478.842124:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878478.842125:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878478.842127:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878478.842129:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878478.842130:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878478.842132:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290067712:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878478.842134:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878478.842135:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878478.842136:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878478.842138:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878478.842140:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.842143:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878478.842145:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:0.0:1788878478.842147:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878478.842156:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878478.842159:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878478.842161:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878478.842162:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.842163:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878478.842165:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.842167:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304090122 00000100:00000001:0.0:1788878478.842168:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304090665 00000100:00000001:0.0:1788878478.842170:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304090122 00000100:00000001:0.0:1788878478.842171:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304090665 00000100:00000001:0.0:1788878478.842173:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878478.842174:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.842175:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878478.842177:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878478/real 1788878478] req@ffff94e50449f800 x1875765290067712/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878478.842189:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878478.842189:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878478.842191:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878478.842192:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878478.842193:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.842194:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878478.842195:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878478.842197:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290067712/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878478.842201:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.842202:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878478.842204:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.842205:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878478.842205:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.842206:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878478.842215:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878478.842217:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878478.842219:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878478.842221:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878478.842223:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290067712:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878478.842225:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.842227:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878478.842227:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878478.842228:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878478.842230:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878478.842231:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878478.842232:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878478.842235:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878478.842236:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878478.842237:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878478.842239:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.842240:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878478.842252:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878478.842253:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878478.842254:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878478.842255:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878478.842256:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878478.842257:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878478.842257:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878478.842258:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878478.842259:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.842260:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878478.842261:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878478.842262:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878478.842263:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878478.842264:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878478.842264:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878478.842265:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878478.842266:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878478.842267:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878478.842268:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.842269:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878478.842271:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878478.842272:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878478.842274:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878478.842275:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878479.609033:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878479.609040:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878479.609046:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878479.609047:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878479.609102:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878479.609105:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878479.609109:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878480.632988:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878480.632994:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878480.632999:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878480.633002:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878480.633005:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878480.633008:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878480.633010:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878480.633012:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878480.633013:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878480.633015:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878480.633035:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878480.633037:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878480.633039:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878481.656985:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878481.656991:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878481.656995:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878481.656996:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878481.657012:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878481.657014:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878481.657016:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878482.680993:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878482.680998:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878482.681004:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878482.681007:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878483.705003:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878483.705014:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878483.705017:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878483.705021:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878483.705029:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878483.705032:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878483.705034:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878483.705037:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878483.705043:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878483.705045:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878483.705072:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878483.705074:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878483.705077:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878483.961028:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878483.961040:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878483.961043:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878483.961048:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878483.961053:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878483.961061:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9422 00000100:00080000:3.0:1788878483.961073:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878483.961076:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878483.961079:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878483.961088:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878483.961091:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878483.961095:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878483.961098:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878483.961102:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878483.961103:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878483.961108:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878483.961112:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878483.961115:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878483.961119:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878483.961122:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961125:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878483.961127:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878483.961141:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878483.961142:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961146:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878483.961149:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878483.961154:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961166:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878483.961177:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290067840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878483.961190:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878483.961196:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961218:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878483.961222:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878483.961226:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878483.961228:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878483.961230:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878483.961234:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878483.961238:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9422 00000100:00080000:3.0:1788878483.961242:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878483.961245:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878483.961247:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878483.961251:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878483.961253:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878483.961255:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878483.961260:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878483.961261:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878483.961264:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878483.961270:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878483.961272:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878483.961281:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1788878483.961285:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878483.961288:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878483.961294:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.961295:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878483.961302:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878483.961307:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878483.961311:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878483.961312:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878483.961321:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878483.961323:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961327:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878483.961333:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878483.961336:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290067840:192.168.203.129@tcp:250:kworker.0 00000008:00000001:3.0:1788878483.961340:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961349:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.961351:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878483.961357:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878483.961360:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878483.961360:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878483.961367:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961368:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878483.961374:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878483.961376:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878483.961387:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878483.961391:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961395:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878483.961405:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290067968/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878483.961415:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878483.961416:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878483.961419:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:3.0:1788878483.961424:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.961430:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878483.961432:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878483.961437:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878483.961467:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878483.961470:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878483.961472:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878483.961476:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878483.961479:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9422 00000100:00080000:3.0:1788878483.961484:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878483.961487:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878483.961490:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878483.961494:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878483.961495:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878483.961498:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878483.961501:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878483.961503:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878483.961505:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878483.961509:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878483.961512:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878483.961514:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878483.961517:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878483.961519:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878483.961521:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961529:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878483.961533:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878483.961537:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304095785 00000008:00000001:3.0:1788878483.961537:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961544:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878483.961551:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878483.961553:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961556:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878483.961559:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788878483.961559:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878483.961566:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.961575:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304095785 00000100:00000001:3.0:1788878483.961576:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.961591:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00100000:3.0:1788878483.961592:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290068096/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878483.961602:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878483.961613:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961628:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878483.961629:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.961640:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878483.961641:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961663:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.961667:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1788878483.961668:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878483.961673:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00080000:3.0:1788878483.961677:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878483.961679:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878483.961682:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878483.961685:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788878483.961687:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878483.961689:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878483.961695:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878483.961696:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878483.961703:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878483.961704:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9422 02000000:00000001:0.0:1788878483.961713:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1788878483.961717:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1788878483.961718:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878483.961725:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788878483.961727:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961730:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:0.0:1788878483.961736:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290067968:192.168.203.129@tcp:38:kworker.0 00000020:00000001:3.0:1788878483.961738:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878483.961745:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878483.961751:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1788878483.961753:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:0.0:1788878483.961756:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1788878483.961758:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:0.0:1788878483.961760:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961765:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788878483.961766:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878483.961772:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:0.0:1788878483.961774:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878483.961780:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878483.961786:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.961787:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788878483.961792:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878483.961793:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 02000000:00000001:3.0:1788878483.961800:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878483.961803:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788878483.961804:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961812:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878483.961815:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878483.961818:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961821:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.961825:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878483.961827:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878483.961830:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.961832:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304095243 00000100:00000001:3.0:1788878483.961833:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878483.961846:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304095785 02000000:00000001:3.0:1788878483.961846:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878483.961854:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.961855:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304095243 00000100:00000001:0.0:1788878483.961862:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304095785 00000100:00000001:0.0:1788878483.961866:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878483.961867:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.961871:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.961875:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878483.961880:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1788878483.961881:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290068224/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878483.961882:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878483.961892:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878483.961896:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878483.961947:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878483.961950:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878483.961956:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878483.961992:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878483.961999:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290068096:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878483.962002:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878483.962006:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878483.962008:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878483.962010:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878483.962014:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.962018:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878483.962019:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:0.0:1788878483.962023:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788878483.962033:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.962036:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304095242 00000100:00000001:0.0:1788878483.962038:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304095785 00000100:00000001:0.0:1788878483.962040:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304095242 00000100:00000001:0.0:1788878483.962041:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304095785 00000100:00000001:0.0:1788878483.962043:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878483.962044:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.962045:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.962047:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878483.962049:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878483.962050:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788878483.962052:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878483.962053:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878483.962053:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878483.962055:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878483.962056:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878483.962057:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878483.962059:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878483.962060:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878483.962063:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290068224:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878483.962065:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878483.962067:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878483.962068:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878483.962069:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878483.962071:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.962074:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878483.962075:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:0.0:1788878483.962078:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788878483.962084:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.962086:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304095242 00000100:00000001:0.0:1788878483.962088:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304095785 00000100:00000001:0.0:1788878483.962090:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304095242 00000100:00000001:0.0:1788878483.962091:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304095785 00000100:00000001:0.0:1788878483.962092:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878483.962093:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.962095:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.962096:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878483.962097:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878483.962098:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788878483.962099:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878483.962099:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878483.962100:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878483.962102:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878484.729078:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878484.729087:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878484.729092:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878484.729094:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878484.729134:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878484.729137:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878484.729140:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878485.753006:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878485.753014:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878485.753021:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878485.753023:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878485.753050:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878485.753053:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878485.753057:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878486.777055:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878486.777065:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878486.777074:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878486.777077:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1788878487.033194:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:2.0:1788878487.033248:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788878487.033269:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788878487.033287:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878487.033313:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878487.033317:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878487.033320:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:2.0:1788878487.033333:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788878487.033338:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878487.033341:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878487.033343:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878487.033344:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:2.0:1788878487.033349:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788878487.033353:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878487.033355:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878487.033356:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878487.033358:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:2.0:1788878487.033364:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788878487.033370:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878487.033373:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878487.033375:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878487.033378:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878487.033413:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878487.033439:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878487.033443:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878487.033459:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878483/real 1788878487] req@ffff94e50449c700 x1875765290068224/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878487.033496:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878487.033499:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878487.033507:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878487.033512:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878487.033514:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033518:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878487.033521:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878487.033529:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290068224/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878487.033541:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033546:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878487.033550:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033552:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878487.033554:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033558:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878487.033576:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878487.033582:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878487.033589:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878487.033593:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878487.033599:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290068224:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878487.033605:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878487.033610:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878483/real 1788878487] req@ffff94e50449c000 x1875765290067968/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878487.033621:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878487.033623:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878487.033627:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878487.033629:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878487.033631:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033634:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878487.033636:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878487.033641:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290067968/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878487.033650:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033653:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878487.033656:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033658:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878487.033659:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033662:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878487.033677:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878487.033681:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878487.033686:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878487.033689:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878487.033693:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290067968:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878487.033699:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878487.033704:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878483/real 1788878487] req@ffff94e50449f480 x1875765290068096/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878487.033716:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878487.033718:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878487.033721:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878487.033724:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878487.033726:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033728:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878487.033730:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878487.033735:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290068096/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878487.033744:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033747:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878487.033749:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033752:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878487.033754:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033756:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878487.033771:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878487.033774:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878487.033779:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878487.033781:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878487.033786:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290068096:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878487.033791:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878487.033796:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878483/real 1788878487] req@ffff94e50449d180 x1875765290067840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878487.033805:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878487.033807:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878487.033811:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878487.033813:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878487.033815:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033817:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878487.033819:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878487.033824:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290067840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878487.033832:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033834:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878487.033837:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033839:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878487.033840:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033842:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878487.033844:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878487.033847:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878487.033851:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878487.033854:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878487.033859:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290067840:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878487.033864:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033867:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878487.033870:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878487.033873:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878487.033880:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878487.033883:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878487.033886:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878487.033891:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878487.033894:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878487.033897:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878487.033899:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033901:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878487.033903:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878487.033904:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878487.033907:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878487.033909:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878487.033910:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878487.033913:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878487.033915:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878487.033958:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878487.033961:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033965:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878487.033966:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878487.033969:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878487.033972:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878487.033974:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878487.033975:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878487.033978:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878487.033980:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878487.033983:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878487.033985:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.033988:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878487.033990:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878487.033991:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878487.033993:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878487.033995:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878487.033997:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878487.033999:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878487.034000:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878487.034003:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878487.034005:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.034008:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878487.034017:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878487.034021:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878487.034024:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878487.034026:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878487.800976:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878487.800983:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878487.800988:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878487.800989:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878487.801017:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878487.801020:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878487.801023:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878488.825020:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878488.825030:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878488.825036:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878488.825037:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878488.825078:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878488.825081:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878488.825083:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878489.081143:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878489.081161:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878489.081166:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878489.081173:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878489.081182:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878489.081194:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9427 00000100:00080000:2.0:1788878489.081211:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878489.081217:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878489.081221:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878489.081233:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878489.081237:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878489.081242:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878489.081246:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878489.081249:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878489.081252:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878489.081260:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878489.081265:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878489.081269:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878489.081273:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878489.081277:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878489.081280:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878489.081284:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878489.081294:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878489.081297:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878489.081302:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878489.081306:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878489.081311:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878489.081326:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878489.081339:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290068352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878489.081353:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878489.081359:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878489.081379:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878489.081386:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878489.081393:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878489.081396:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878489.081400:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878489.081428:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878489.081436:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9427 00000100:00080000:2.0:1788878489.081445:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878489.081450:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878489.081454:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878489.081457:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788878489.081461:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878489.081467:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878489.081469:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788878489.081472:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878489.081477:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788878489.081480:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878489.081485:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878489.081489:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878489.081493:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878489.081496:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:0.0:1788878489.081509:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1788878489.081510:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:0.0:1788878489.081520:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878489.081521:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878489.081529:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878489.081534:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878489.081542:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788878489.081553:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290068352:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878489.081553:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878489.081563:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878489.081572:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:2.0:1788878489.081576:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788878489.081583:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:2.0:1788878489.081586:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878489.081593:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878489.081595:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878489.081604:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878489.081610:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878489.081616:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878489.081617:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878489.081621:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878489.081627:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878489.081631:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878489.081645:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.081658:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878489.081664:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00000001:0.0:1788878489.081672:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1788878489.081686:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290068480/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878489.081707:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878489.081714:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878489.081721:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878489.081727:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788878489.081738:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788878489.081740:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878489.081749:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878489.081753:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878489.081761:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878489.081763:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788878489.081770:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9427 00000100:00000001:0.0:1788878489.081777:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788878489.081783:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878489.081785:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878489.081793:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878489.081795:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878489.081800:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878489.081810:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878489.081813:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878489.081821:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878489.081825:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304100905 00000020:00000001:2.0:1788878489.081827:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878489.081837:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878489.081842:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878489.081844:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878489.081847:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1788878489.081854:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878489.081862:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.081863:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304100905 02000000:00000001:2.0:1788878489.081871:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878489.081881:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1788878489.081883:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878489.081894:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.081895:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878489.081899:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878489.081956:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788878489.081966:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878489.081975:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000008:00000001:2.0:1788878489.081976:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878489.081984:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878489.081996:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878489/real 1788878489] req@ffff94e5045e7480 x1875765290068352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788878489.082002:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878489.082008:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878489.082017:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878489.082024:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878489.082031:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788878489.082037:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878489.082044:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878489.082047:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.082053:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878489.082062:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00100000:2.0:1788878489.082065:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290068608/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878489.082068:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.082079:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878489.082087:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878489.082099:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878489.082110:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878489.082111:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290068352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878489.082118:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878489.082128:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878489.082144:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878489.082151:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878489.082153:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878489.082156:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878489.082167:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878489.082168:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878489.082178:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9427 00000100:00000001:0.0:1788878489.082182:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878489.082190:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878489.082191:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878489.082200:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878489.082201:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878489.082208:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878489.082211:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000020:00000001:2.0:1788878489.082218:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878489.082220:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1788878489.082225:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00080000:0.0:1788878489.082233:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:2.0:1788878489.082233:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878489.082245:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878489.082254:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00080000:0.0:1788878489.082255:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878489.082262:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878489.082266:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:2.0:1788878489.082277:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1788878489.082287:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290068352:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878489.082289:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878489.082297:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878489.082307:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878489.082310:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878489.082316:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.082320:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878489.082324:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878489.082329:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000008:00100000:2.0:1788878489.082331:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788878489.082338:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000008:00000001:2.0:1788878489.082340:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878489.082348:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878489.082351:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:2.0:1788878489.082356:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000020:00000001:0.0:1788878489.082358:0:2361:0:(genops.c:1216:class_import_put()) Process entered 02000000:00000001:2.0:1788878489.082363:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878489.082366:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:2.0:1788878489.082371:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878489.082377:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878489.082378:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878489.082382:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878489.082385:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878489.082394:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878489.082397:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.082398:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.082435:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878489.082444:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290068736/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878489.082461:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878489.082469:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878489.082480:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878489.082482:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878489.082489:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878489.082491:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.082496:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878489.082503:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878489.082504:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.082507:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878489.082517:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878489.082522:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878489.082527:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878489.082536:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290068480:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878489.082545:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878489.082549:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878489.082552:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878489.082556:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878489.082562:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.082575:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878489.082578:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:0.0:1788878489.082587:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878489.082616:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878489.082626:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878489.082631:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878489.082635:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.082638:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878489.082644:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.082652:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304100364 00000100:00000001:0.0:1788878489.082657:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304100905 00000100:00000001:0.0:1788878489.082661:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304100364 00000100:00000001:0.0:1788878489.082666:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304100905 00000100:00000001:0.0:1788878489.082670:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878489.082674:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.082678:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878489.082687:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878489/real 1788878489] req@ffff94e5045e4700 x1875765290068480/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878489.082705:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878489.082708:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878489.082713:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878489.082717:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878489.082719:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.082723:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878489.082726:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878489.082735:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290068480/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878489.082753:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.082758:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878489.082763:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.082767:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878489.082769:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.082773:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878489.082797:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878489.082804:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878489.082812:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878489.082816:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878489.082824:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290068480:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878489.082832:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878489.082835:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878489.082838:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878489.082843:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878489.082847:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878489.082851:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878489.082858:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290068608:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878489.082864:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878489.082868:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878489.082871:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878489.082874:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878489.082881:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.082892:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878489.082897:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:0.0:1788878489.082939:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878489.082968:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878489.082977:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878489.082982:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878489.082985:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.082988:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878489.082993:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.083000:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304100363 00000100:00000001:0.0:1788878489.083005:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304100905 00000100:00000001:0.0:1788878489.083008:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304100363 00000100:00000001:0.0:1788878489.083012:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304100905 00000100:00000001:0.0:1788878489.083016:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878489.083020:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.083024:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878489.083032:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878489/real 1788878489] req@ffff94e5045e6680 x1875765290068608/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878489.083063:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878489.083066:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878489.083071:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878489.083075:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878489.083077:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.083081:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878489.083084:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878489.083093:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290068608/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878489.083105:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.083110:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878489.083114:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.083117:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878489.083120:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.083124:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878489.083145:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878489.083150:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878489.083156:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878489.083161:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878489.083166:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290068608:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878489.083173:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.083177:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878489.083180:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878489.083183:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878489.083187:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878489.083189:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878489.083192:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878489.083196:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878489.083198:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878489.083202:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878489.083204:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.083208:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878489.083211:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878489.083213:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878489.083216:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878489.083218:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878489.083220:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878489.083222:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878489.083224:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878489.083227:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878489.083229:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.083232:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.083238:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878489.083241:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.083244:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878489.083247:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878489.083250:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878489.083252:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878489.083255:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878489.083258:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878489.083262:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878489.083265:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878489.083271:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290068736:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878489.083278:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878489.083281:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878489.083284:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878489.083287:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878489.083293:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.083303:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878489.083306:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:0.0:1788878489.083312:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878489.083336:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878489.083345:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878489.083350:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878489.083352:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.083356:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878489.083361:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.083368:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304100363 00000100:00000001:0.0:1788878489.083373:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304100905 00000100:00000001:0.0:1788878489.083378:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304100363 00000100:00000001:0.0:1788878489.083382:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304100905 00000100:00000001:0.0:1788878489.083387:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878489.083391:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.083394:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878489.083403:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878489/real 1788878489] req@ffff94e5045e6a00 x1875765290068736/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878489.083449:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878489.083451:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878489.083458:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878489.083462:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878489.083465:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.083469:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878489.083472:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878489.083481:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290068736/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878489.083496:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.083500:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878489.083504:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.083508:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878489.083510:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.083513:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878489.083534:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878489.083540:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878489.083546:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878489.083551:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878489.083558:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290068736:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878489.083566:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.083570:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878489.083573:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878489.083576:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878489.083581:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878489.083584:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878489.083587:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878489.083591:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878489.083594:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878489.083598:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878489.083600:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.083603:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878489.083610:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878489.083613:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878489.083617:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878489.083620:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878489.848986:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878489.848994:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878489.849001:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878489.849003:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878489.849061:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878489.849065:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878489.849070:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878490.873050:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878490.873058:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878490.873064:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878490.873067:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878490.873071:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878490.873075:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878490.873078:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878490.873080:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878490.873083:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878490.873086:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878491.897011:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878491.897021:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878491.897030:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878491.897032:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878491.897084:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878491.897087:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878491.897091:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878492.920985:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878492.920991:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878492.920996:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878492.920997:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878492.921031:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878492.921033:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878492.921035:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878493.945025:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878493.945036:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878493.945040:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878493.945045:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878493.945053:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878493.945057:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878493.945061:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878493.945065:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878493.945072:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878493.945075:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878493.945158:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878493.945163:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878493.945179:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878494.200994:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878494.201002:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878494.201005:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878494.201008:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878494.201012:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878494.201017:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9433 00000100:00080000:3.0:1788878494.201037:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878494.201039:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878494.201041:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878494.201048:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878494.201050:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878494.201053:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878494.201054:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878494.201056:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878494.201057:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878494.201060:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878494.201062:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878494.201064:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878494.201066:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878494.201068:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878494.201076:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878494.201078:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878494.201087:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878494.201088:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878494.201090:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878494.201092:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878494.201096:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878494.201116:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878494.201122:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290068864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878494.201130:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878494.201133:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878494.201143:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878494.201145:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878494.201150:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878494.201151:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878494.201152:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878494.201154:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878494.201157:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9433 00000100:00080000:3.0:1788878494.201159:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878494.201161:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878494.201162:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878494.201165:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878494.201165:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878494.201167:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878494.201169:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788878494.201169:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878494.201173:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878494.201174:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878494.201175:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788878494.201177:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878494.201179:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878494.201181:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201182:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788878494.201183:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878494.201186:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878494.201189:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878494.201191:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878494.201193:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878494.201195:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878494.201196:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:0.0:1788878494.201208:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788878494.201210:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878494.201215:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290068864:192.168.203.129@tcp:250:kworker.0 00000008:00000001:3.0:1788878494.201215:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878494.201217:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878494.201222:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878494.201223:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878494.201225:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878494.201227:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878494.201227:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878494.201228:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878494.201232:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878494.201233:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878494.201236:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878494.201238:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1788878494.201242:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290068992/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788878494.201245:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878494.201255:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878494.201257:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878494.201259:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878494.201261:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878494.201263:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878494.201264:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878494.201266:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788878494.201266:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878494.201271:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:3.0:1788878494.201287:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878494.201288:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788878494.201293:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9433 00000100:00080000:3.0:1788878494.201297:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878494.201299:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878494.201300:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878494.201303:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878494.201304:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878494.201305:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878494.201307:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878494.201308:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878494.201309:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878494.201311:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878494.201313:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878494.201314:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878494.201316:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878494.201318:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878494.201319:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878494.201321:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1788878494.201325:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00000001:3.0:1788878494.201325:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878494.201329:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878494.201333:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878494.201334:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878494.201336:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878494.201337:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878494.201338:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788878494.201339:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201344:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878494.201345:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201347:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201351:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:3.0:1788878494.201353:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290069120/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878494.201355:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201361:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304106025 00000100:00000001:3.0:1788878494.201380:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878494.201384:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201385:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878494.201387:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878494.201392:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878494.201405:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304106025 00000100:00080000:3.0:1788878494.201406:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878494.201411:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878494.201413:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788878494.201414:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878494.201419:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878494.201420:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878494.201422:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201425:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788878494.201426:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9433 00000100:00100000:0.0:1788878494.201433:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878494/real 1788878494] req@ffff94e50449ce00 x1875765290068864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878494.201433:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878494.201439:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878494.201442:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878494.201450:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000001:0.0:1788878494.201451:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1788878494.201453:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000001:0.0:1788878494.201454:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1788878494.201468:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878494.201470:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000020:00000001:3.0:1788878494.201473:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878494.201474:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878494.201477:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878494.201479:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878494.201480:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878494.201484:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878494.201484:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:0.0:1788878494.201491:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290068864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878494.201493:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878494.201499:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201505:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878494.201505:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878494.201509:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878494.201511:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.201512:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878494.201513:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.201516:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:3.0:1788878494.201517:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878494.201518:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878494.201533:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878494.201534:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788878494.201540:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00080000:0.0:1788878494.201544:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000008:00100000:3.0:1788878494.201545:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878494.201550:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000001:3.0:1788878494.201552:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878494.201557:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878494.201560:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290068864:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788878494.201563:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878494.201570:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878494.201571:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878494.201575:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878494.201576:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788878494.201577:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878494.201581:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878494.201581:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878494.201585:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788878494.201586:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878494.201589:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878494.201593:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00100000:3.0:1788878494.201593:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290069248/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:0.0:1788878494.201596:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878494.201601:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878494.201604:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878494.201604:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878494.201608:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201609:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878494.201612:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878494.201613:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201617:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.201625:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878494.201626:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201628:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878494.201630:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878494.201631:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878494.201633:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878494.201634:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878494.201636:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878494.201637:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878494.201639:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878494.201642:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290068992:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878494.201645:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878494.201646:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878494.201647:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878494.201648:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878494.201651:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201656:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878494.201657:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:0.0:1788878494.201661:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878494.201671:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878494.201674:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878494.201676:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878494.201677:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201678:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878494.201680:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201683:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304105483 00000100:00000001:0.0:1788878494.201685:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304106025 00000100:00000001:0.0:1788878494.201687:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304105483 00000100:00000001:0.0:1788878494.201689:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304106025 00000100:00000001:0.0:1788878494.201690:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878494.201691:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201692:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878494.201695:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878494/real 1788878494] req@ffff94e50449d880 x1875765290068992/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878494.201701:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878494.201702:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878494.201703:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878494.201704:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878494.201705:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.201707:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878494.201708:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878494.201710:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290068992/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878494.201715:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.201716:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878494.201718:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.201719:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878494.201720:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.201721:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878494.201730:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878494.201732:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878494.201734:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878494.201735:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878494.201738:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290068992:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878494.201740:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878494.201749:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878494.201750:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878494.201752:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878494.201756:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878494.201757:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878494.201759:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290069120:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878494.201762:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878494.201764:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878494.201766:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878494.201768:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878494.201771:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201776:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878494.201778:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:0.0:1788878494.201781:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878494.201793:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878494.201797:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878494.201800:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878494.201802:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201803:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878494.201806:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201810:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304105481 00000100:00000001:0.0:1788878494.201813:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304106025 00000100:00000001:0.0:1788878494.201815:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304105481 00000100:00000001:0.0:1788878494.201817:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304106025 00000100:00000001:0.0:1788878494.201819:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878494.201821:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201822:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878494.201825:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878494/real 1788878494] req@ffff94e50449c380 x1875765290069120/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878494.201836:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878494.201837:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878494.201839:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878494.201840:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878494.201841:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.201842:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878494.201843:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878494.201846:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290069120/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878494.201850:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.201852:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878494.201854:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.201855:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878494.201855:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.201856:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878494.201865:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878494.201866:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878494.201868:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878494.201870:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878494.201872:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290069120:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878494.201874:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878494.201875:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878494.201876:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878494.201878:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878494.201879:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878494.201880:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878494.201882:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290069248:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878494.201884:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878494.201885:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878494.201886:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878494.201887:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878494.201889:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201893:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878494.201894:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:0.0:1788878494.201896:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878494.201928:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878494.201932:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878494.201933:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878494.201934:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201936:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878494.201937:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201940:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304105482 00000100:00000001:0.0:1788878494.201941:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304106025 00000100:00000001:0.0:1788878494.201943:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304105482 00000100:00000001:0.0:1788878494.201944:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304106025 00000100:00000001:0.0:1788878494.201946:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878494.201947:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.201948:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878494.201951:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878494/real 1788878494] req@ffff94e50449fb80 x1875765290069248/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878494.201956:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878494.201956:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878494.201958:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878494.201959:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878494.201960:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.201961:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878494.201962:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878494.201964:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290069248/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878494.201968:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.201970:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878494.201972:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.201973:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878494.201974:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.201975:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878494.201983:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878494.201985:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878494.201987:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878494.201989:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878494.201991:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290069248:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878494.201993:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.201994:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878494.201995:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878494.201996:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878494.201997:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878494.201998:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878494.201999:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878494.202001:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878494.202002:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878494.202004:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878494.202005:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.202007:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878494.202008:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878494.202009:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878494.202011:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878494.202012:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878494.202013:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878494.202015:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878494.202017:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878494.202018:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878494.202019:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.202021:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878494.202022:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878494.202023:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878494.202025:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878494.202026:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878494.202028:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878494.202029:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878494.202031:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878494.202033:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878494.202034:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.202036:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878494.202040:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878494.202042:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878494.202044:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878494.202046:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878494.969024:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878494.969029:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878494.969033:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878494.969034:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878495.993029:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878495.993038:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878495.993045:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878495.993047:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878495.993086:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878495.993090:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878495.993093:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878497.017037:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878497.017045:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878497.017053:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878497.017055:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878497.017082:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878497.017085:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878497.017089:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878498.041073:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878498.041084:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878498.041094:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878498.041098:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878498.041170:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878498.041177:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878498.041184:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878499.064993:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878499.065000:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878499.065005:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878499.065006:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878499.321021:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878499.321030:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878499.321033:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878499.321036:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878499.321040:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878499.321047:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9438 00000100:00080000:3.0:1788878499.321056:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878499.321058:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878499.321060:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878499.321085:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878499.321088:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878499.321091:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878499.321094:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878499.321096:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878499.321098:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878499.321104:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878499.321109:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878499.321113:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878499.321118:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878499.321122:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321125:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878499.321128:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878499.321141:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878499.321160:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321165:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878499.321169:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878499.321175:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321190:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878499.321205:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290069376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878499.321220:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878499.321225:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321263:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878499.321268:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878499.321273:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878499.321275:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878499.321279:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878499.321283:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878499.321288:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9438 00000100:00080000:3.0:1788878499.321296:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878499.321299:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878499.321301:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878499.321304:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788878499.321307:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878499.321312:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788878499.321313:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878499.321321:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788878499.321322:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878499.321328:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1788878499.321330:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878499.321333:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321378:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788878499.321380:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878499.321382:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:0.0:1788878499.321384:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878499.321389:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878499.321390:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878499.321395:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878499.321398:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290069376:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788878499.321398:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878499.321406:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878499.321407:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878499.321411:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878499.321411:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878499.321417:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321417:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:0.0:1788878499.321422:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:3.0:1788878499.321424:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878499.321427:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878499.321428:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321430:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878499.321439:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878499.321440:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321441:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878499.321443:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878499.321444:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321450:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.321455:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00100000:3.0:1788878499.321456:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290069504/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878499.321459:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000100:00000001:0.0:1788878499.321468:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878499.321471:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878499.321474:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321478:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878499.321480:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878499.321484:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878499.321485:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878499.321486:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878499.321489:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878499.321491:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9438 00000100:00080000:3.0:1788878499.321495:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878499.321497:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878499.321498:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878499.321501:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878499.321504:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:0.0:1788878499.321505:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:3.0:1788878499.321507:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878499.321511:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878499.321513:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878499.321515:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878499.321519:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:3.0:1788878499.321520:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878499.321525:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878499.321525:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878499.321527:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878499.321530:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878499.321532:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878499.321533:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788878499.321538:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.321542:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321542:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878499.321546:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878499.321549:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304111145 00000008:00000001:3.0:1788878499.321550:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321557:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878499.321561:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878499.321562:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321563:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878499.321565:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788878499.321565:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878499.321568:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.321573:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304111145 00000100:00000001:3.0:1788878499.321574:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878499.321583:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290069632/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878499.321584:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878499.321591:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878499.321595:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878499.321596:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321600:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.321602:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788878499.321604:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878499.321607:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878499.321610:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878499.321611:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878499.321612:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878499.321614:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878499.321618:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9438 00000100:00100000:0.0:1788878499.321620:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878499/real 1788878499] req@ffff94e50449e680 x1875765290069376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878499.321621:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878499.321625:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878499.321628:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878499.321634:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878499.321637:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878499.321641:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000400:00000001:0.0:1788878499.321643:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1788878499.321644:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000400:00000001:0.0:1788878499.321647:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878499.321649:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878499.321652:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878499.321654:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:3.0:1788878499.321656:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878499.321660:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878499.321661:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.321665:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878499.321666:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878499.321670:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878499.321671:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878499.321675:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878499.321676:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321680:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878499.321683:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878499.321684:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290069376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1788878499.321687:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321692:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878499.321698:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878499.321702:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878499.321703:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321708:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878499.321710:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878499.321710:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878499.321714:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.321716:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.321719:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878499.321720:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.321721:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.321725:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878499.321730:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:3.0:1788878499.321730:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290069760/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878499.321738:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878499.321748:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878499.321749:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878499.321751:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878499.321753:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.321760:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878499.321765:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290069376:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878499.321771:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.321773:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878499.321775:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878499.321778:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878499.321782:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878499.321784:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878499.321785:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878499.321789:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878499.321790:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878499.321792:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878499.321793:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.321795:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.321802:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878499.321803:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.321805:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878499.321806:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878499.321807:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878499.321809:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878499.321811:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878499.321812:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878499.321814:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878499.321816:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878499.321818:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290069504:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878499.321821:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878499.321822:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878499.321824:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878499.321825:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878499.321827:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.321832:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878499.321834:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:0.0:1788878499.321838:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878499.321848:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878499.321852:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878499.321854:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878499.321855:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.321856:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878499.321859:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.321862:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304110603 00000100:00000001:0.0:1788878499.321865:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304111145 00000100:00000001:0.0:1788878499.321867:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304110603 00000100:00000001:0.0:1788878499.321869:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304111145 00000100:00000001:0.0:1788878499.321871:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878499.321872:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.321874:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878499.321877:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878499/real 1788878499] req@ffff94e50449df80 x1875765290069504/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878499.321884:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878499.321885:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878499.321886:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878499.321888:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878499.321889:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.321891:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878499.321892:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878499.321895:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290069504/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878499.321922:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.321924:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878499.321927:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.321928:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878499.321929:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.321931:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878499.321942:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878499.321944:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878499.321947:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878499.321949:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878499.321952:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290069504:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878499.321955:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878499.321956:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878499.321958:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878499.321959:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878499.321961:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878499.321963:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878499.321965:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290069632:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878499.321968:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878499.321969:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878499.321970:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878499.321971:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878499.321974:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.321978:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878499.321980:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:0.0:1788878499.321982:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878499.321991:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878499.321995:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878499.321997:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878499.321998:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.322000:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878499.322002:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.322005:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304110602 00000100:00000001:0.0:1788878499.322007:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304111145 00000100:00000001:0.0:1788878499.322009:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304110602 00000100:00000001:0.0:1788878499.322010:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304111145 00000100:00000001:0.0:1788878499.322012:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878499.322014:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.322015:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878499.322018:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878499/real 1788878499] req@ffff94e50449fb80 x1875765290069632/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878499.322024:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878499.322025:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878499.322027:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878499.322029:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878499.322030:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.322031:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878499.322032:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878499.322035:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290069632/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878499.322040:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.322041:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878499.322043:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.322044:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878499.322045:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.322047:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878499.322056:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878499.322058:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878499.322061:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878499.322063:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878499.322065:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290069632:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878499.322068:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878499.322069:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878499.322070:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878499.322072:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878499.322073:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878499.322075:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878499.322077:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290069760:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878499.322079:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878499.322080:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878499.322081:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878499.322083:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878499.322085:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.322088:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878499.322089:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:0.0:1788878499.322092:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878499.322099:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878499.322102:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878499.322103:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878499.322105:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.322106:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878499.322108:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.322110:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304110602 00000100:00000001:0.0:1788878499.322112:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304111145 00000100:00000001:0.0:1788878499.322114:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304110602 00000100:00000001:0.0:1788878499.322116:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304111145 00000100:00000001:0.0:1788878499.322118:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878499.322119:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.322120:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878499.322124:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878499/real 1788878499] req@ffff94e50449c380 x1875765290069760/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878499.322130:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878499.322131:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878499.322133:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878499.322134:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878499.322135:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.322136:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878499.322137:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878499.322140:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290069760/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878499.322156:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.322157:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878499.322159:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.322160:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878499.322161:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.322162:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878499.322172:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878499.322174:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878499.322176:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878499.322177:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878499.322180:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290069760:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878499.322182:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.322184:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878499.322185:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878499.322186:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878499.322188:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878499.322189:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878499.322190:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878499.322192:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878499.322193:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878499.322195:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878499.322196:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.322198:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878499.322199:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878499.322200:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878499.322201:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878499.322202:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878499.322203:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878499.322204:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878499.322205:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878499.322206:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878499.322207:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.322208:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878499.322209:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878499.322210:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878499.322211:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878499.322212:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878499.322213:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878499.322214:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878499.322215:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878499.322216:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878499.322217:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.322218:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878499.322220:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878499.322221:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878499.322223:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878499.322225:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878500.089032:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878500.089039:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878500.089044:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878500.089047:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878500.089049:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878500.089052:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878500.089054:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878500.089056:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878500.089058:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878500.089059:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878500.089078:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878500.089080:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878500.089083:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878501.113012:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878501.113018:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878501.113025:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878501.113027:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878501.113067:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878501.113070:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878501.113074:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878502.137019:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878502.137026:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878502.137030:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878502.137031:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878502.137048:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878502.137051:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878502.137055:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878503.161005:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878503.161011:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878503.161013:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878503.161016:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878503.161020:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878503.161022:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878503.161024:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878503.161025:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878503.161028:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878503.161029:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878504.185077:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878504.185084:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878504.185091:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878504.185093:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878504.185135:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878504.185138:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878504.185142:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878504.441035:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878504.441048:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878504.441051:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878504.441056:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878504.441061:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878504.441070:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9443 00000100:00080000:2.0:1788878504.441083:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878504.441087:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878504.441090:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878504.441100:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878504.441103:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878504.441109:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878504.441112:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878504.441115:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878504.441117:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878504.441123:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878504.441127:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878504.441131:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878504.441136:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878504.441140:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878504.441143:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878504.441146:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878504.441159:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878504.441162:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878504.441165:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878504.441169:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878504.441174:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878504.441188:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878504.441201:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290069888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878504.441216:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878504.441221:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878504.441242:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878504.441247:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878504.441251:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878504.441253:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878504.441255:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878504.441259:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878504.441264:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9443 00000100:00080000:2.0:1788878504.441270:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878504.441274:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878504.441277:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878504.441283:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788878504.441283:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878504.441289:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878504.441290:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788878504.441294:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878504.441295:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878504.441298:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878504.441316:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878504.441319:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:0.0:1788878504.441323:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788878504.441326:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788878504.441327:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878504.441331:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:0.0:1788878504.441332:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878504.441341:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290069888:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1788878504.441343:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878504.441351:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788878504.441352:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878504.441356:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878504.441358:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878504.441360:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878504.441364:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878504.441368:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878504.441370:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878504.441378:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878504.441382:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878504.441388:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878504.441391:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000008:00000001:2.0:1788878504.441392:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.441395:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:2.0:1788878504.441398:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.441401:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1788878504.441408:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878504.441413:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878504.441416:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878504.441418:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878504.441422:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878504.441431:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878504.441439:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290070016/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878504.441444:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878504.441457:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878504.441459:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878504.441463:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.441466:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878504.441470:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878504.441472:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.441473:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878504.441479:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878504.441481:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878504.441486:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304116265 00000100:00080000:2.0:1788878504.441489:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878504.441493:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878504.441496:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878504.441499:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878504.441500:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878504.441508:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9443 00000100:00080000:2.0:1788878504.441514:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878504.441518:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304116265 00000100:00000001:2.0:1788878504.441518:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878504.441524:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878504.441527:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1788878504.441531:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878504.441533:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878504.441538:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878504.441538:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878504.441544:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:2.0:1788878504.441545:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00100000:0.0:1788878504.441552:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878504/real 1788878504] req@ffff94e5045e5880 x1875765290069888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788878504.441553:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878504.441562:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878504.441566:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:0.0:1788878504.441575:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1788878504.441578:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000400:00000001:0.0:1788878504.441579:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878504.441584:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878504.441586:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.441589:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:2.0:1788878504.441592:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878504.441593:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.441597:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878504.441600:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878504.441600:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878504.441609:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878504.441610:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290069888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878504.441614:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878504.441619:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878504.441628:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.441630:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878504.441633:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.441636:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878504.441640:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878504.441642:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878504.441643:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878504.441648:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878504.441649:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.441651:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878504.441654:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878504.441655:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788878504.441658:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00080000:0.0:1788878504.441660:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1788878504.441663:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878504.441667:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878504.441672:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788878504.441674:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878504.441676:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290069888:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878504.441685:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.441688:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1788878504.441688:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290070144/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878504.441698:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878504.441702:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788878504.441706:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1788878504.441708:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878504.441712:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:2.0:1788878504.441713:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878504.441715:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:2.0:1788878504.441721:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878504.441722:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878504.441726:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:2.0:1788878504.441729:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878504.441730:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878504.441733:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.441739:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878504.441741:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878504.441745:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878504.441747:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1788878504.441748:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878504.441750:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.441756:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878504.441759:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878504.441760:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878504.441765:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878504.441767:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788878504.441769:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9443 00000100:00000001:0.0:1788878504.441770:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878504.441774:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878504.441778:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878504.441781:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878504.441782:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878504.441787:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290070016:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878504.441789:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878504.441794:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878504.441796:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878504.441798:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878504.441800:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788878504.441803:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:0.0:1788878504.441805:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878504.441815:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878504.441817:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1788878504.441819:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878504.441820:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 00000020:00000001:2.0:1788878504.441823:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878504.441827:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1788878504.441827:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878504.441832:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878504.441834:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878504.441839:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878504.441844:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788878504.441845:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:2.0:1788878504.441849:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878504.441853:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:2.0:1788878504.441854:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878504.441857:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788878504.441860:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.441861:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878504.441864:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878504.441865:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000008:00100000:2.0:1788878504.441869:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878504.441871:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788878504.441877:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.441879:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304115723 00000100:00000001:2.0:1788878504.441881:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.441886:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304116265 02000000:00000001:2.0:1788878504.441892:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878504.441895:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.441898:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304115723 00000100:00000001:2.0:1788878504.441899:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878504.441937:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304116265 00000100:00000001:0.0:1788878504.441940:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878504.441942:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.441944:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878504.441950:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878504/real 1788878504] req@ffff94e5045e5180 x1875765290070016/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788878504.441951:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878504.441957:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788878504.441966:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878504.441967:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878504.441971:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.441972:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878504.441975:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878504.441978:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.441981:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878504.441984:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878504.441985:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290070272/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878504.441991:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290070016/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878504.442018:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878504.442023:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878504.442025:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.442030:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878504.442035:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.442037:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442042:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878504.442044:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442047:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878504.442068:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878504.442072:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878504.442076:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878504.442079:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878504.442082:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290070016:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878504.442085:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878504.442087:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878504.442088:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878504.442090:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878504.442092:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878504.442094:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878504.442096:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290070144:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878504.442099:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878504.442101:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878504.442102:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878504.442104:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878504.442107:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.442113:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878504.442115:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:0.0:1788878504.442118:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878504.442130:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878504.442135:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878504.442137:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878504.442138:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.442140:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878504.442142:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.442145:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304115722 00000100:00000001:0.0:1788878504.442147:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304116265 00000100:00000001:0.0:1788878504.442150:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304115722 00000100:00000001:0.0:1788878504.442152:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304116265 00000100:00000001:0.0:1788878504.442154:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878504.442155:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.442157:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878504.442160:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878504/real 1788878504] req@ffff94e5045e4a80 x1875765290070144/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878504.442167:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878504.442168:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878504.442170:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878504.442172:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878504.442173:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442175:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878504.442176:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878504.442179:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290070144/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878504.442185:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442187:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878504.442189:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442190:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878504.442192:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442193:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878504.442202:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878504.442204:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878504.442206:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878504.442208:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878504.442211:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290070144:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878504.442214:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442216:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878504.442217:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878504.442219:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878504.442221:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878504.442222:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878504.442223:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878504.442225:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878504.442226:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878504.442228:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878504.442229:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442231:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878504.442232:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878504.442233:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878504.442234:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878504.442235:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878504.442236:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878504.442238:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878504.442239:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878504.442240:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878504.442241:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442242:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442245:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878504.442246:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.442248:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878504.442249:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878504.442250:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878504.442251:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878504.442252:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878504.442254:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878504.442255:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878504.442257:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878504.442259:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290070272:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878504.442262:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878504.442264:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878504.442265:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878504.442266:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878504.442268:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.442272:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878504.442273:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:0.0:1788878504.442276:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878504.442285:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878504.442288:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878504.442290:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878504.442291:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.442292:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878504.442294:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.442297:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304115722 00000100:00000001:0.0:1788878504.442316:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304116265 00000100:00000001:0.0:1788878504.442320:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304115722 00000100:00000001:0.0:1788878504.442322:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304116265 00000100:00000001:0.0:1788878504.442324:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878504.442326:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.442327:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878504.442332:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878504/real 1788878504] req@ffff94e5045e7100 x1875765290070272/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878504.442339:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878504.442340:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878504.442343:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878504.442345:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878504.442346:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442347:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878504.442349:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878504.442352:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290070272/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878504.442358:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442360:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878504.442361:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442363:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878504.442364:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442365:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878504.442375:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878504.442377:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878504.442380:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878504.442382:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878504.442385:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290070272:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878504.442388:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442389:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878504.442390:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878504.442391:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878504.442393:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878504.442394:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878504.442395:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878504.442398:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878504.442399:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878504.442400:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878504.442401:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442403:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878504.442405:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878504.442407:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878504.442408:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878504.442410:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878505.208992:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878505.208998:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878505.209003:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878505.209005:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878505.209027:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878505.209029:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878505.209032:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878506.232996:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878506.233005:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878506.233013:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878506.233015:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878506.233040:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878506.233044:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878506.233048:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878507.256999:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878507.257005:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878507.257009:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878507.257011:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878508.281028:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878508.281033:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878508.281037:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878508.281038:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878508.281059:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878508.281061:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878508.281063:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878509.305000:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878509.305006:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878509.305009:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878509.305010:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878509.305057:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878509.305059:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878509.305062:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878509.561023:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878509.561035:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878509.561039:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878509.561043:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878509.561049:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878509.561074:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9448 00000100:00080000:0.0:1788878509.561082:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878509.561085:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878509.561086:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878509.561093:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878509.561095:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878509.561097:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878509.561099:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878509.561101:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878509.561102:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878509.561105:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878509.561107:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878509.561109:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878509.561112:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878509.561114:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561116:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878509.561135:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878509.561148:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878509.561150:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561154:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878509.561158:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878509.561164:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561177:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878509.561203:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978700 x1875765290070400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878509.561211:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878509.561214:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561229:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878509.561231:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878509.561233:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878509.561234:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878509.561236:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878509.561238:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878509.561239:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9448 00000100:00080000:0.0:1788878509.561242:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878509.561243:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878509.561244:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878509.561246:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878509.561247:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878509.561249:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788878509.561250:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878509.561251:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878509.561252:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878509.561254:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878509.561255:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878509.561257:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878509.561261:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878509.561276:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561279:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878509.561282:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878509.561285:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561287:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878509.561290:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878509.561292:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561293:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878509.561294:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878509.561295:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.561297:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878509.561300:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.561303:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788878509.561306:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290070528/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878509.561307:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878509.561311:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878509.561313:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561316:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878509.561319:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878509.561319:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878509.561323:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878509.561324:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878509.561327:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878509.561328:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878509.561333:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1788878509.561334:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290070400:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878509.561335:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878509.561338:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878509.561345:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788878509.561345:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878509.561350:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1788878509.561351:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9448 02000000:00000001:3.0:1788878509.561353:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878509.561357:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1788878509.561358:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1788878509.561362:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561363:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878509.561366:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878509.561371:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878509.561373:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878509.561375:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878509.561378:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878509.561380:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878509.561381:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878509.561383:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878509.561385:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00080000:0.0:1788878509.561390:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878509.561392:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788878509.561397:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878509.561399:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878509.561402:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878509.561404:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561405:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878509.561407:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878509.561409:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561410:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878509.561413:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878509.561414:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561415:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878509.561417:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878509.561418:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561422:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878509.561426:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897aa00 x1875765290070656/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878509.561433:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878509.561452:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561458:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788878509.561466:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1788878509.561467:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878509.561476:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878509.561479:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878509.561482:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878509.561487:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788878509.561489:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:0.0:1788878509.561495:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9448 00000100:00000001:3.0:1788878509.561500:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1788878509.561504:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878509.561507:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561530:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878509.561532:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878509.561533:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878509.561539:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878509.561540:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878509.561543:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788878509.561544:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304121385 00000020:00000001:0.0:1788878509.561545:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788878509.561550:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878509.561552:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878509.561553:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878509.561556:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878509.561556:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878509.561558:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878509.561561:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878509.561563:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878509.561564:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304121385 02000000:00000001:0.0:1788878509.561565:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561569:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878509.561571:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788878509.561571:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000008:00000001:0.0:1788878509.561574:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.561576:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878509.561577:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.561599:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878509.561601:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788878509.561602:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1788878509.561603:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561606:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878509.561607:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878509.561609:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878509.561611:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878509/real 1788878509] req@ffff94e508978700 x1875765290070400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878509.561613:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878509.561621:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290070784/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788878509.561627:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788878509.561630:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:3.0:1788878509.561631:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878509.561634:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878509.561637:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.561637:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878509.561641:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878509.561644:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.561664:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878509.561668:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878509.561681:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290070400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878509.561698:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.561706:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878509.561727:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.561729:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878509.561731:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.561733:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878509.561735:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878509.561738:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878509.561743:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878509.561746:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878509.561750:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290070400:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878509.561755:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.561757:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878509.561759:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878509.561761:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878509.561764:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878509.561765:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878509.561767:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878509.561769:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878509.561771:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878509.561772:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878509.561773:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.561775:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.561781:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878509.561782:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.561784:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878509.561785:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878509.561786:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878509.561787:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878509.561788:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878509.561790:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878509.561791:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878509.561793:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878509.561795:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290070528:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878509.561797:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878509.561799:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878509.561800:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878509.561801:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878509.561804:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.561808:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878509.561809:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:3.0:1788878509.561813:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878509.561822:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878509.561825:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878509.561827:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878509.561828:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.561829:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878509.561831:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.561834:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304120843 00000100:00000001:3.0:1788878509.561836:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304121385 00000100:00000001:3.0:1788878509.561838:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304120843 00000100:00000001:3.0:1788878509.561847:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304121385 00000100:00000001:3.0:1788878509.561849:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878509.561853:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.561854:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878509.561857:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878509/real 1788878509] req@ffff94e50897a300 x1875765290070528/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878509.561863:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878509.561864:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878509.561866:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878509.561867:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878509.561868:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.561869:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878509.561870:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878509.561872:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290070528/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878509.561876:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.561878:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878509.561880:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.561881:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878509.561882:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.561883:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878509.561892:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878509.561894:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878509.561896:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878509.561897:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878509.561899:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290070528:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878509.561924:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878509.561925:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878509.561926:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878509.561927:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878509.561929:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878509.561930:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878509.561932:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290070656:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878509.561934:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878509.561936:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878509.561937:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878509.561938:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878509.561940:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.561944:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878509.561945:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:3.0:1788878509.561947:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878509.561954:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878509.561957:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878509.561959:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878509.561960:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.561961:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878509.561963:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.561965:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304120842 00000100:00000001:3.0:1788878509.561967:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304121385 00000100:00000001:3.0:1788878509.561968:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304120842 00000100:00000001:3.0:1788878509.561970:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304121385 00000100:00000001:3.0:1788878509.561971:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878509.561973:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.561974:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878509.561976:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878509/real 1788878509] req@ffff94e50897aa00 x1875765290070656/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878509.561981:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878509.561982:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878509.561983:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878509.561984:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878509.561986:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.561987:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878509.561988:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878509.561990:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290070656/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878509.561994:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.561995:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878509.561997:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.561998:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878509.561999:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.562000:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878509.562007:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878509.562009:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878509.562011:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878509.562012:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878509.562014:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290070656:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878509.562016:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878509.562017:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878509.562019:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878509.562020:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878509.562021:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878509.562022:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878509.562024:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290070784:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878509.562026:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878509.562027:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878509.562028:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878509.562029:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878509.562030:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.562032:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878509.562033:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:3.0:1788878509.562035:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878509.562041:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878509.562043:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878509.562044:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878509.562045:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.562046:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878509.562048:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.562050:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304120842 00000100:00000001:3.0:1788878509.562051:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304121385 00000100:00000001:3.0:1788878509.562053:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304120842 00000100:00000001:3.0:1788878509.562054:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304121385 00000100:00000001:3.0:1788878509.562056:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878509.562057:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.562058:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878509.562060:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878509/real 1788878509] req@ffff94e508978a80 x1875765290070784/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878509.562065:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878509.562065:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878509.562066:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878509.562067:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878509.562068:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.562069:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878509.562070:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878509.562073:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290070784/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878509.562077:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.562079:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878509.562080:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.562081:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878509.562082:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.562083:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878509.562090:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878509.562092:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878509.562093:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878509.562094:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878509.562096:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290070784:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878509.562098:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.562099:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878509.562100:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878509.562101:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878509.562102:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878509.562103:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878509.562104:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878509.562106:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878509.562107:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878509.562109:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878509.562110:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.562111:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878509.562112:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878509.562112:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878509.562114:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878509.562114:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878509.562115:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878509.562116:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878509.562117:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878509.562118:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878509.562119:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.562120:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878509.562121:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878509.562122:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878509.562123:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878509.562124:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878509.562124:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878509.562125:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878509.562126:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878509.562127:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878509.562128:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.562129:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878509.562130:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878509.562131:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878509.562132:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878509.562133:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878510.328982:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878510.328987:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878510.328991:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878510.328993:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878510.328995:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878510.328998:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878510.329000:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878510.329001:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878510.329003:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878510.329005:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878510.329023:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878510.329025:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878510.329027:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878511.352993:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878511.352998:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878511.353004:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878511.353005:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878512.377052:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878512.377060:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878512.377065:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878512.377067:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878512.377090:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878512.377093:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878512.377097:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878513.400991:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878513.400998:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878513.401000:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878513.401004:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878513.401008:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878513.401011:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878513.401013:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878513.401015:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878513.401018:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878513.401019:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878513.401045:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878513.401048:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878513.401050:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878514.424990:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878514.424996:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878514.424999:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878514.425000:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878514.425017:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878514.425019:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878514.425021:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878514.681149:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878514.681163:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878514.681169:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878514.681175:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878514.681185:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878514.681205:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9453 00000100:00080000:3.0:1788878514.681253:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878514.681260:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878514.681264:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878514.681278:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878514.681284:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878514.681290:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878514.681296:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878514.681301:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878514.681304:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878514.681313:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878514.681318:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878514.681324:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878514.681331:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878514.681338:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878514.681343:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878514.681348:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878514.681364:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878514.681367:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878514.681373:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878514.681377:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878514.681384:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878514.681402:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878514.681419:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290070912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878514.681440:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878514.681448:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878514.681478:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878514.681485:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878514.681492:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878514.681496:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878514.681499:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878514.681507:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878514.681514:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9453 00000100:00080000:3.0:1788878514.681523:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878514.681528:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878514.681532:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878514.681541:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878514.681546:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878514.681550:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878514.681558:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788878514.681558:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878514.681567:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878514.681568:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878514.681574:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878514.681576:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878514.681581:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878514.681587:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878514.681591:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:0.0:1788878514.681595:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878514.681603:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878514.681603:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878514.681612:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878514.681621:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290070912:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878514.681628:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878514.681633:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878514.681636:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878514.681640:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878514.681646:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.681666:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878514.681668:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:0.0:1788878514.681674:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878514.681718:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878514.681733:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788878514.681737:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878514.681741:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878514.681746:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.681749:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788878514.681751:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878514.681755:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878514.681760:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.681764:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304126505 00000100:00000001:3.0:1788878514.681768:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878514.681778:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878514.681779:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00000001:3.0:1788878514.681791:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.681795:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304126505 00000100:00000001:3.0:1788878514.681799:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.681805:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1788878514.681812:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878514.681813:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878514.681821:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.681822:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.681833:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788878514.681835:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878514.681840:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1788878514.681843:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878514/real 1788878514] req@ffff94e50449d880 x1875765290070912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788878514.681847:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878514.681865:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788878514.681867:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878514.681873:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878514.681879:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878514.681885:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00100000:3.0:1788878514.681887:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290071040/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878514.681889:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.681896:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878514.681899:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878514.681950:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290070912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878514.681968:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.681974:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878514.681982:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.681987:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878514.681991:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.681997:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878514.682001:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878514.682008:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878514.682018:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878514.682025:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878514.682032:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290070912:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878514.682041:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.682044:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878514.682047:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878514.682051:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878514.682057:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878514.682061:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878514.682064:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878514.682071:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878514.682074:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878514.682080:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878514.682083:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.682087:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.682100:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878514.682104:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.682108:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878514.682111:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878514.682169:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878514.682177:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878514.682192:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878514.682199:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878514.682208:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878514.682213:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878514.682217:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878514.682226:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878514.682253:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878514.682263:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788878514.682263:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9453 00000100:00000001:0.0:1788878514.682270:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878514.682276:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878514.682281:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878514.682283:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1788878514.682287:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878514.682294:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878514.682297:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788878514.682300:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878514.682304:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:0.0:1788878514.682311:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290071040:192.168.203.129@tcp:38:kworker.0 00000020:00000001:3.0:1788878514.682318:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878514.682320:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1788878514.682328:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1788878514.682329:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878514.682336:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878514.682340:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 02000000:00000001:0.0:1788878514.682341:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878514.682349:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878514.682352:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878514.682361:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878514.682366:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878514.682367:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878514.682372:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:0.0:1788878514.682381:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788878514.682384:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878514.682394:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878514.682400:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878514.682408:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:0.0:1788878514.682413:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788878514.682418:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.682424:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878514.682427:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878514.682432:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00100000:3.0:1788878514.682436:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878514.682440:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788878514.682445:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.682449:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878514.682453:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.682459:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878514.682467:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878514.682472:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304125964 00000100:00000001:0.0:1788878514.682482:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304126505 00000100:00000001:0.0:1788878514.682487:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304125964 00000100:00000001:0.0:1788878514.682492:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304126505 00000100:00000001:0.0:1788878514.682497:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878514.682501:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.682504:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878514.682514:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878514/real 1788878514] req@ffff94e50449ce00 x1875765290071040/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878514.682533:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878514.682537:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878514.682542:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878514.682546:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878514.682549:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.682553:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878514.682555:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878514.682561:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290071040/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878514.682573:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878514.682573:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.682581:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878514.682586:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878514.682588:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.682597:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788878514.682597:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878514.682603:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878514.682607:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.682608:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878514.682628:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.682629:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878514.682640:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878514.682650:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1788878514.682653:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290071168/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878514.682654:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878514.682665:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290071040:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878514.682680:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.682687:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878514.682691:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878514.682698:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788878514.682699:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1788878514.682703:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878514.682710:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:3.0:1788878514.682711:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878514.682715:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878514.682722:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1788878514.682725:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878514.682726:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878514.682732:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878514.682738:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788878514.682738:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878514.682746:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788878514.682753:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878514.682756:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878514.682761:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878514.682762:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.682770:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1788878514.682771:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878514.682773:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878514.682778:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878514.682783:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878514.682787:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878514.682787:0:177900:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:0.0:1788878514.682795:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1788878514.682800:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9453 02000000:00000001:0.0:1788878514.682802:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878514.682810:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878514.682814:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878514.682817:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290071168:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878514.682822:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878514.682829:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788878514.682830:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:0.0:1788878514.682835:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1788878514.682839:0:177900:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:0.0:1788878514.682840:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878514.682847:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1788878514.682848:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1788878514.682854:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:0.0:1788878514.682856:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878514.682862:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878514.682867:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878514.682870:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878514.682871:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878514.682876:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00080000:3.0:1788878514.682885:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878514.682887:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878514.682890:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878514.682895:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878514.682937:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:0.0:1788878514.682947:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788878514.682948:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878514.682956:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878514.682961:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878514.682963:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00000001:3.0:1788878514.682971:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.682973:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878514.682976:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.682980:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.682985:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788878514.682987:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878514.682991:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878514.682993:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.683001:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304125963 00000100:00000001:3.0:1788878514.683001:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878514.683004:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878514.683011:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.683013:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304126505 00000100:00000001:0.0:1788878514.683021:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304125963 00000100:00000001:3.0:1788878514.683024:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.683026:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304126505 00000100:00000001:0.0:1788878514.683034:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878514.683039:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878514.683042:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290071296/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878514.683044:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878514.683056:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878514/real 1788878514] req@ffff94e50449d180 x1875765290071168/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878514.683069:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878514.683077:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788878514.683089:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788878514.683092:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788878514.683094:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878514.683103:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878514.683107:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878514.683110:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.683113:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878514.683116:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878514.683122:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290071168/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878514.683132:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.683136:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878514.683139:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.683141:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878514.683143:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.683146:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878514.683158:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878514.683161:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878514.683165:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878514.683168:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878514.683172:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290071168:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878514.683177:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.683179:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878514.683180:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878514.683182:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878514.683184:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878514.683186:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878514.683187:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878514.683190:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878514.683192:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878514.683194:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878514.683196:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.683198:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.683201:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878514.683203:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.683206:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878514.683208:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878514.683210:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878514.683211:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878514.683213:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878514.683216:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878514.683219:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878514.683222:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878514.683225:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290071296:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878514.683254:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878514.683257:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878514.683259:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878514.683262:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878514.683267:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.683274:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878514.683276:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:0.0:1788878514.683281:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878514.683297:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878514.683303:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878514.683306:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878514.683308:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.683310:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878514.683313:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.683318:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304125963 00000100:00000001:0.0:1788878514.683321:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304126505 00000100:00000001:0.0:1788878514.683324:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304125963 00000100:00000001:0.0:1788878514.683326:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304126505 00000100:00000001:0.0:1788878514.683329:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878514.683332:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.683334:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878514.683339:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878514/real 1788878514] req@ffff94e50449f480 x1875765290071296/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878514.683348:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878514.683350:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878514.683353:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878514.683355:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878514.683356:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.683359:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878514.683360:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878514.683366:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290071296/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878514.683374:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.683377:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878514.683380:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.683382:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878514.683383:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.683386:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878514.683397:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878514.683400:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878514.683403:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878514.683406:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878514.683409:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290071296:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878514.683414:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.683416:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878514.683418:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878514.683420:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878514.683422:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878514.683424:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878514.683426:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878514.683428:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878514.683431:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878514.683434:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878514.683437:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.683440:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878514.683445:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878514.683448:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878514.683452:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878514.683455:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878515.449039:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878515.449048:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878515.449053:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878515.449055:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878516.473043:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878516.473054:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878516.473065:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878516.473069:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878516.473115:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878516.473121:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878516.473127:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878517.497041:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878517.497048:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878517.497054:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878517.497055:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878517.497087:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878517.497090:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878517.497093:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878518.521022:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878518.521030:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878518.521036:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878518.521038:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878518.521063:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878518.521066:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878518.521070:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878519.545028:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878519.545052:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878519.545072:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878519.545074:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878519.545244:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878519.545251:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878519.545256:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878519.801047:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878519.801065:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878519.801070:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878519.801077:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878519.801085:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878519.801096:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9458 00000100:00080000:2.0:1788878519.801109:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878519.801114:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878519.801117:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878519.801141:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878519.801144:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878519.801149:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878519.801152:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878519.801155:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878519.801157:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878519.801175:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878519.801179:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878519.801183:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878519.801187:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878519.801191:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878519.801214:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878519.801217:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878519.801232:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878519.801234:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878519.801250:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878519.801254:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878519.801260:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878519.801275:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878519.801287:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290071424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878519.801302:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878519.801307:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878519.801328:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878519.801344:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878519.801349:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878519.801351:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878519.801354:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878519.801358:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878519.801361:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9458 00000100:00080000:2.0:1788878519.801366:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878519.801369:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878519.801371:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878519.801375:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878519.801377:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878519.801380:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878519.801384:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878519.801387:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878519.801390:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878519.801395:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878519.801397:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878519.801399:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878519.801404:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788878519.801408:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878519.801412:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788878519.801414:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.801418:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878519.801420:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878519.801425:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:2.0:1788878519.801428:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878519.801430:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788878519.801438:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878519.801442:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788878519.801444:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878519.801453:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878519.801454:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878519.801459:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878519.801467:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878519.801467:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878519.801472:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878519.801479:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878519.801488:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290071424:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878519.801490:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.801505:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:2.0:1788878519.801507:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290071552/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788878519.801516:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878519.801524:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878519.801534:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788878519.801542:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878519.801554:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878519.801558:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878519.801559:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878519.801567:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878519.801571:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878519.801574:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878519.801577:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878519.801581:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878519.801585:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788878519.801587:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9458 00000100:00000001:0.0:1788878519.801592:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000100:00080000:2.0:1788878519.801597:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878519.801603:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878519.801604:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878519.801610:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878519.801615:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878519.801617:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878519.801620:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878519.801623:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878519.801626:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878519.801628:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878519.801632:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878519.801635:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878519.801638:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878519.801642:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878519.801645:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788878519.801660:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878519.801661:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878519.801668:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878519.801672:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878519.801675:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.801680:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:2.0:1788878519.801684:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878519.801689:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788878519.801690:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.801695:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878519.801697:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878519.801702:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:2.0:1788878519.801704:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878519.801711:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.801712:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878519.801719:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.801721:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304131625 00000100:00100000:2.0:1788878519.801736:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290071680/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878519.801741:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878519.801755:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304131625 00000100:00000001:2.0:1788878519.801756:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878519.801765:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.801769:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878519.801770:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878519.801782:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878519.801783:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878519.801794:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878519.801796:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878519.801801:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878519.801802:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878519.801808:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878519.801817:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878519/real 1788878519] req@ffff94e5045e7800 x1875765290071424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878519.801818:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878519.801829:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9458 00000100:00080000:2.0:1788878519.801838:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878519.801847:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000400:00000001:0.0:1788878519.801849:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788878519.801851:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:0.0:1788878519.801855:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1788878519.801859:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878519.801862:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000020:00000001:2.0:1788878519.801864:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878519.801869:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000020:00000001:2.0:1788878519.801870:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878519.801875:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788878519.801878:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878519.801881:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878519.801884:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878519.801887:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878519.801890:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:0.0:1788878519.801899:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290071424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878519.801950:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878519.801955:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878519.801962:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878519.801968:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878519.801978:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878519.801987:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878519.801995:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1788878519.801996:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878519.802007:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802008:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878519.802010:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878519.802022:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878519.802024:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878519.802027:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802031:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878519.802034:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878519.802036:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878519.802040:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878519.802042:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1788878519.802046:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802048:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878519.802058:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788878519.802059:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878519.802073:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290071808/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878519.802079:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878519.802093:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788878519.802098:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878519.802106:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878519.802109:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290071424:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878519.802111:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802131:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802137:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878519.802141:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878519.802147:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878519.802155:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878519.802159:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878519.802173:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878519.802178:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878519.802181:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878519.802184:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878519.802186:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802189:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802221:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878519.802223:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802227:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878519.802229:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878519.802231:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878519.802233:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878519.802236:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878519.802239:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878519.802242:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878519.802245:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878519.802250:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290071552:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878519.802255:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878519.802258:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878519.802260:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878519.802262:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878519.802267:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802276:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878519.802278:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 00000100:00000001:0.0:1788878519.802284:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878519.802305:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878519.802312:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878519.802316:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878519.802318:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802320:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878519.802324:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802329:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304131084 00000100:00000001:0.0:1788878519.802333:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304131625 00000100:00000001:0.0:1788878519.802336:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304131084 00000100:00000001:0.0:1788878519.802339:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304131625 00000100:00000001:0.0:1788878519.802342:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878519.802344:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802347:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878519.802352:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878519/real 1788878519] req@ffff94e5045e6300 x1875765290071552/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878519.802363:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878519.802365:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878519.802367:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878519.802370:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878519.802372:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802374:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878519.802376:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878519.802380:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290071552/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878519.802389:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802392:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878519.802394:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802396:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878519.802398:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802400:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878519.802415:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878519.802418:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878519.802422:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878519.802426:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878519.802430:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290071552:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878519.802434:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878519.802436:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878519.802438:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878519.802441:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878519.802443:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878519.802447:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878519.802453:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290071680:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878519.802458:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878519.802462:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878519.802465:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878519.802468:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878519.802473:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802483:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878519.802485:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:0.0:1788878519.802491:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878519.802515:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878519.802524:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878519.802529:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878519.802532:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802535:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878519.802541:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802547:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304131082 00000100:00000001:0.0:1788878519.802560:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304131625 00000100:00000001:0.0:1788878519.802564:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304131082 00000100:00000001:0.0:1788878519.802568:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304131625 00000100:00000001:0.0:1788878519.802571:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878519.802574:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802577:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878519.802583:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878519/real 1788878519] req@ffff94e5045e5500 x1875765290071680/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878519.802603:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878519.802604:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878519.802609:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878519.802611:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878519.802613:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802615:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878519.802617:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878519.802622:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290071680/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878519.802630:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802633:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878519.802636:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802638:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878519.802639:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802642:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878519.802653:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878519.802656:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878519.802660:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878519.802662:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878519.802666:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290071680:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878519.802671:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878519.802673:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878519.802675:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878519.802678:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878519.802680:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878519.802682:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878519.802686:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290071808:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878519.802690:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878519.802692:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878519.802695:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878519.802697:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878519.802701:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802707:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878519.802709:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:0.0:1788878519.802714:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878519.802729:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878519.802735:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878519.802737:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878519.802739:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802741:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878519.802744:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802748:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304131082 00000100:00000001:0.0:1788878519.802751:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304131625 00000100:00000001:0.0:1788878519.802754:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304131082 00000100:00000001:0.0:1788878519.802757:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304131625 00000100:00000001:0.0:1788878519.802759:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878519.802761:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802763:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878519.802768:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878519/real 1788878519] req@ffff94e5045e5c00 x1875765290071808/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878519.802777:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878519.802778:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878519.802781:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878519.802783:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878519.802785:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802787:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878519.802788:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878519.802793:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290071808/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878519.802800:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802803:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878519.802806:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802808:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878519.802810:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802812:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878519.802824:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878519.802827:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878519.802831:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878519.802834:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878519.802838:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290071808:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878519.802842:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802844:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878519.802846:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878519.802848:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878519.802850:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878519.802852:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878519.802854:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878519.802857:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878519.802858:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878519.802861:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878519.802863:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802865:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878519.802866:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878519.802868:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878519.802870:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878519.802872:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878519.802873:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878519.802875:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878519.802876:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878519.802878:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878519.802879:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802881:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878519.802883:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878519.802884:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878519.802886:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878519.802887:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878519.802888:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878519.802890:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878519.802892:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878519.802894:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878519.802895:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802898:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878519.802932:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878519.802935:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878519.802938:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878519.802940:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878520.568953:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878520.568958:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878520.568962:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878520.568964:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878520.568966:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878520.568969:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878520.568971:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878520.568973:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878520.568975:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878520.568976:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878520.569035:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878520.569038:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878520.569041:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878521.593009:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878521.593016:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878521.593022:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878521.593024:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878521.593074:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878521.593077:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878521.593081:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878522.617045:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878522.617064:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878522.617072:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878522.617074:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878522.617100:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878522.617103:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878522.617106:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878523.641047:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878523.641056:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878523.641060:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878523.641064:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878523.641071:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878523.641074:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878523.641076:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878523.641080:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878523.641084:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878523.641086:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878523.641259:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878523.641268:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878523.641275:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878524.665009:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878524.665017:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878524.665024:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878524.665026:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878524.665150:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878524.665155:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878524.665184:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878524.921036:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878524.921046:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878524.921048:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878524.921052:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878524.921057:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878524.921065:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9463 00000100:00080000:3.0:1788878524.921076:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878524.921079:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878524.921082:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878524.921090:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878524.921092:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878524.921096:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878524.921098:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878524.921101:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878524.921102:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878524.921107:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878524.921109:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878524.921113:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878524.921116:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878524.921119:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878524.921121:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878524.921124:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878524.921136:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878524.921138:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878524.921141:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878524.921144:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878524.921149:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878524.921188:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878524.921200:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290071936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878524.921222:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878524.921226:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878524.921243:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878524.921246:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878524.921250:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878524.921252:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878524.921254:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878524.921257:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878524.921260:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9463 00000100:00080000:3.0:1788878524.921264:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878524.921267:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878524.921268:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878524.921272:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878524.921274:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878524.921277:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878524.921279:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878524.921281:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878524.921283:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878524.921287:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878524.921289:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878524.921291:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878524.921294:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788878524.921298:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878524.921301:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788878524.921303:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.921307:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878524.921308:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878524.921314:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1788878524.921316:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878524.921319:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788878524.921321:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878524.921327:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878524.921329:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878524.921333:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878524.921338:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878524.921338:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878524.921343:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878524.921356:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878524.921367:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1788878524.921370:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290071936:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788878524.921372:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878524.921381:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.921385:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1788878524.921392:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290072064/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788878524.921395:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878524.921402:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878524.921406:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1788878524.921410:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878524.921411:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878524.921418:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878524.921421:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1788878524.921423:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878524.921427:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878524.921431:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878524.921433:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878524.921437:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878524.921440:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9463 00000100:00000001:0.0:1788878524.921444:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788878524.921444:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878524.921446:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878524.921473:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:3.0:1788878524.921474:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878524.921485:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878524.921486:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1788878524.921490:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878524.921507:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878524.921510:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878524.921513:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878524.921515:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878524.921519:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878524.921523:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878524.921526:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878524.921530:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878524.921533:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878524.921537:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878524.921540:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878524.921543:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878524.921545:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788878524.921554:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788878524.921555:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878524.921561:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878524.921565:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878524.921567:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878524.921571:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.921575:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878524.921578:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.921585:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1788878524.921591:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290072192/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878524.921594:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.921604:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878524.921613:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878524.921614:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878524.921621:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.921623:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304136745 00000100:00000001:3.0:1788878524.921626:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878524.921634:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878524.921638:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878524.921640:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878524.921643:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878524.921644:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878524.921648:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878524.921655:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9463 00000100:00000001:0.0:1788878524.921658:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304136745 00000100:00080000:3.0:1788878524.921662:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878524.921669:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878524.921670:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878524.921675:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878524.921680:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788878524.921682:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878524.921688:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878524.921690:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878524.921693:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878524.921697:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1788878524.921700:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878524.921706:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1788878524.921712:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878524/real 1788878524] req@ffff94e50449c000 x1875765290071936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878524.921712:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878524.921725:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878524.921737:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878524.921744:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878524.921753:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:0.0:1788878524.921754:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1788878524.921758:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788878524.921760:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878524.921765:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878524.921769:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000008:00100000:3.0:1788878524.921771:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878524.921776:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000008:00000001:3.0:1788878524.921777:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878524.921798:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878524.921802:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878524.921803:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878524.921805:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878524.921806:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878524.921808:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.921812:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878524.921813:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.921821:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:3.0:1788878524.921826:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290072320/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878524.921827:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878524.921842:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290071936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878524.921846:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878524.921854:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.921864:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878524.921866:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.921869:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878524.921876:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.921878:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878524.921880:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.921883:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878524.921885:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878524.921888:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878524.921893:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878524.921897:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878524.921966:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290071936:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878524.921975:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.921988:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878524.921991:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878524.921994:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878524.921999:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878524.922002:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878524.922004:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878524.922009:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878524.922011:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878524.922014:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878524.922016:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922018:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922026:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878524.922029:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.922032:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878524.922034:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878524.922037:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878524.922039:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878524.922041:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878524.922045:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878524.922047:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878524.922051:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878524.922057:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290072064:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878524.922063:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878524.922066:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878524.922069:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878524.922072:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878524.922077:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.922087:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878524.922090:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:0.0:1788878524.922097:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878524.922118:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878524.922124:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878524.922127:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878524.922128:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.922130:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878524.922133:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.922138:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304136204 00000100:00000001:0.0:1788878524.922141:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304136745 00000100:00000001:0.0:1788878524.922143:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304136204 00000100:00000001:0.0:1788878524.922146:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304136745 00000100:00000001:0.0:1788878524.922148:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878524.922150:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.922152:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878524.922156:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878524/real 1788878524] req@ffff94e50449c700 x1875765290072064/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878524.922201:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878524.922202:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878524.922206:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878524.922208:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878524.922210:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922212:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878524.922214:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878524.922218:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290072064/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878524.922226:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922228:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878524.922231:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922232:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878524.922234:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922235:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878524.922247:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878524.922249:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878524.922253:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878524.922255:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878524.922258:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290072064:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878524.922262:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878524.922263:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878524.922265:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878524.922268:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878524.922270:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878524.922271:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878524.922274:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290072192:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878524.922278:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878524.922280:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878524.922282:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878524.922284:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878524.922287:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.922293:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878524.922295:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:0.0:1788878524.922299:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878524.922312:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878524.922317:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878524.922319:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878524.922321:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.922323:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878524.922327:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.922332:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304136202 00000100:00000001:0.0:1788878524.922336:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304136745 00000100:00000001:0.0:1788878524.922339:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304136202 00000100:00000001:0.0:1788878524.922343:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304136745 00000100:00000001:0.0:1788878524.922347:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878524.922361:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.922364:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878524.922371:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878524/real 1788878524] req@ffff94e50449f800 x1875765290072192/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878524.922385:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878524.922387:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878524.922392:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878524.922394:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878524.922397:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922399:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878524.922419:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878524.922427:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290072192/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878524.922440:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922443:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878524.922447:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922450:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878524.922451:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922454:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878524.922477:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878524.922480:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878524.922485:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878524.922487:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878524.922491:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290072192:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878524.922496:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878524.922498:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878524.922500:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878524.922503:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878524.922505:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878524.922508:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878524.922512:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290072320:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878524.922516:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878524.922518:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878524.922520:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878524.922522:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878524.922527:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.922534:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878524.922536:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:0.0:1788878524.922541:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878524.922558:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878524.922564:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878524.922567:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878524.922569:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.922571:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878524.922575:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.922579:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304136202 00000100:00000001:0.0:1788878524.922582:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304136745 00000100:00000001:0.0:1788878524.922585:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304136202 00000100:00000001:0.0:1788878524.922588:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304136745 00000100:00000001:0.0:1788878524.922591:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878524.922604:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.922606:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878524.922611:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878524/real 1788878524] req@ffff94e50449ca80 x1875765290072320/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878524.922620:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878524.922621:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878524.922624:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878524.922626:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878524.922628:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922630:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878524.922632:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878524.922637:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290072320/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878524.922645:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922647:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878524.922650:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922651:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878524.922653:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922655:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878524.922666:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878524.922669:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878524.922673:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878524.922676:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878524.922680:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290072320:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878524.922684:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922686:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878524.922688:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878524.922690:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878524.922693:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878524.922696:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878524.922697:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878524.922700:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878524.922702:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878524.922705:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878524.922706:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922709:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878524.922710:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878524.922712:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878524.922714:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878524.922716:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878524.922717:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878524.922719:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878524.922721:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878524.922723:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878524.922724:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922726:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878524.922728:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878524.922729:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878524.922731:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878524.922732:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878524.922733:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878524.922735:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878524.922736:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878524.922738:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878524.922739:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922741:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878524.922746:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878524.922748:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878524.922750:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878524.922752:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878525.688973:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878525.688978:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878525.688982:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878525.688983:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878525.689001:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878525.689004:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878525.689006:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878526.713018:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878526.713026:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878526.713033:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878526.713035:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878526.713076:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878526.713080:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878526.713084:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878527.033065:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878527.033073:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878527.033076:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878527.033083:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878527.033088:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878527.033093:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878527.033096:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878527.033100:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878527.033104:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878527.737021:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878527.737032:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878527.737041:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878527.737044:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878527.737262:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878527.737268:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878527.737273:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878528.057000:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878528.057011:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878528.057015:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878528.057024:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878528.057030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878528.057036:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878528.057040:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878528.057045:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878528.057050:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878528.761024:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878528.761032:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878528.761041:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878528.761044:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878528.761126:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878528.761151:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878528.761158:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878529.081021:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878529.081028:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878529.081030:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878529.081037:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878529.081041:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878529.081046:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878529.081051:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878529.081054:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878529.081058:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878529.785060:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878529.785067:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878529.785073:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878529.785075:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878529.785175:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878529.785180:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878529.785184:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878530.041196:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878530.041209:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878530.041212:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878530.041217:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878530.041222:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878530.041231:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9468 00000100:00080000:2.0:1788878530.041244:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878530.041247:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878530.041250:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878530.041259:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878530.041263:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878530.041267:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878530.041271:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878530.041273:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878530.041275:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878530.041280:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878530.041283:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878530.041287:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878530.041292:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878530.041295:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878530.041298:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878530.041301:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878530.041315:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878530.041317:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878530.041320:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878530.041324:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878530.041330:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878530.041343:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878530.041355:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5f80 x1875765290072448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878530.041368:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878530.041373:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878530.041404:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878530.041409:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878530.041413:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878530.041415:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878530.041417:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878530.041420:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878530.041424:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9468 00000100:00080000:2.0:1788878530.041428:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878530.041431:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878530.041434:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878530.041441:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878530.041445:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878530.041449:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878530.041454:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878530.041458:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878530.041462:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878530.041462:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878530.041470:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878530.041477:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1788878530.041479:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878530.041482:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878530.041488:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878530.041491:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878530.041496:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878530.041501:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878530.041502:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878530.041510:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878530.041517:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788878530.041519:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290072448:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878530.041529:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.041532:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788878530.041539:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:0.0:1788878530.041541:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878530.041548:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788878530.041551:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788878530.041552:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:2.0:1788878530.041559:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878530.041561:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878530.041563:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878530.041575:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878530.041577:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878530.041580:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878530.041582:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878530.041586:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788878530.041587:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.041593:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074556288 : -117763634995328 : ffff94e5045e5f80) 00000100:00000001:0.0:1788878530.041601:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878530.041602:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878530.041614:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4380 x1875765290072576/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878530.041625:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878530.041628:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878530.041633:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878530.041637:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878530.041642:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878530.041644:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878530.041646:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878530.041654:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000800:00000100:0.0:1788878530.041655:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788878530.041665:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9468 00000100:00000001:0.0:1788878530.041675:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788878530.041677:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878530.041683:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878530.041686:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878530.041693:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878530.041700:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878530.041702:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878530.041707:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:2.0:1788878530.041710:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878530.041716:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878530.041719:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878530.041724:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304141865 00000020:00000001:2.0:1788878530.041728:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878530.041737:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878530.041740:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878530.041742:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878530.041751:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304141865 00000100:00080000:2.0:1788878530.041756:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878530.041764:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878530.041766:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878530.041775:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878530.041781:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878530.041785:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.041787:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878530.041788:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878530.041794:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878530.041799:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788878530.041800:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878530.041809:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.041810:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1788878530.041818:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878530.041824:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878530.041829:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878530/real 1788878530] req@ffff94e5045e5f80 x1875765290072448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878530.041829:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878530.041835:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878530.041840:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878530.041851:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788878530.041867:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:2.0:1788878530.041868:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290072704/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1788878530.041872:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878530.041881:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878530.041884:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878530.041888:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878530.041890:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.041893:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878530.041897:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.041899:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1788878530.041949:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878530.041956:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878530.041958:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878530.041963:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878530.041966:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878530.041970:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5f80 x1875765290072448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878530.041972:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878530.041981:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9468 00000100:00080000:2.0:1788878530.041992:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878530.042001:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878530.042002:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878530.042004:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878530.042015:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878530.042017:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1788878530.042018:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878530.042024:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878530.042032:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788878530.042032:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878530.042044:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878530.042044:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878530.042054:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878530.042054:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878530.042067:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1788878530.042067:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878530.042076:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878530.042077:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878530.042082:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878530.042087:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1788878530.042091:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1788878530.042092:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878530.042100:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878530.042105:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1788878530.042109:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000008:00000001:2.0:1788878530.042112:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042115:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788878530.042118:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878530.042151:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:0.0:1788878530.042153:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290072448:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878530.042156:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878530.042166:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878530.042169:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878530.042170:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878530.042177:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042178:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878530.042185:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878530.042189:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788878530.042190:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878530.042195:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878530.042202:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00100000:2.0:1788878530.042204:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290072832/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:0.0:1788878530.042206:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878530.042214:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878530.042219:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878530.042221:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878530.042224:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878530.042229:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042230:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042237:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878530.042238:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042247:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878530.042252:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042256:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878530.042258:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878530.042260:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878530.042263:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878530.042265:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878530.042269:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878530.042272:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878530.042275:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878530.042283:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290072576:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878530.042290:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878530.042293:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878530.042296:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878530.042299:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878530.042305:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042317:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878530.042320:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074549120 : -117763635002496 : ffff94e5045e4380) 00000100:00000001:0.0:1788878530.042328:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878530.042357:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878530.042366:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878530.042371:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878530.042374:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042377:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878530.042382:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042390:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304141324 00000100:00000001:0.0:1788878530.042396:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304141865 00000100:00000001:0.0:1788878530.042401:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304141324 00000100:00000001:0.0:1788878530.042405:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304141865 00000100:00000001:0.0:1788878530.042410:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878530.042413:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042417:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878530.042426:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878530/real 1788878530] req@ffff94e5045e4380 x1875765290072576/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878530.042441:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878530.042443:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878530.042447:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878530.042450:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878530.042451:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042454:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878530.042455:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878530.042460:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4380 x1875765290072576/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878530.042469:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042473:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878530.042476:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042478:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878530.042479:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042481:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878530.042495:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878530.042497:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878530.042501:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878530.042503:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878530.042507:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290072576:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878530.042512:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878530.042513:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878530.042515:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878530.042518:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878530.042521:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878530.042523:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878530.042527:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290072704:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878530.042531:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878530.042534:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878530.042536:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878530.042538:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878530.042542:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042549:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878530.042551:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:0.0:1788878530.042556:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878530.042572:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878530.042578:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878530.042580:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878530.042582:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042584:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878530.042587:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042591:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304141322 00000100:00000001:0.0:1788878530.042594:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304141865 00000100:00000001:0.0:1788878530.042597:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304141322 00000100:00000001:0.0:1788878530.042600:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304141865 00000100:00000001:0.0:1788878530.042602:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878530.042604:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042606:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878530.042611:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878530/real 1788878530] req@ffff94e5045e5c00 x1875765290072704/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878530.042620:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878530.042621:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878530.042624:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878530.042625:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878530.042627:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042629:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878530.042631:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878530.042635:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290072704/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878530.042651:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042654:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878530.042656:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042658:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878530.042660:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042661:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878530.042672:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878530.042675:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878530.042678:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878530.042680:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878530.042683:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290072704:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878530.042688:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878530.042689:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878530.042691:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878530.042693:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878530.042696:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878530.042698:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878530.042701:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290072832:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878530.042704:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878530.042706:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878530.042707:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878530.042709:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878530.042712:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042717:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878530.042718:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:0.0:1788878530.042722:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878530.042732:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878530.042736:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878530.042738:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878530.042739:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042742:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878530.042745:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042748:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304141322 00000100:00000001:0.0:1788878530.042751:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304141865 00000100:00000001:0.0:1788878530.042753:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304141322 00000100:00000001:0.0:1788878530.042756:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304141865 00000100:00000001:0.0:1788878530.042759:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878530.042763:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042766:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878530.042773:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878530/real 1788878530] req@ffff94e5045e5500 x1875765290072832/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878530.042789:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878530.042791:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878530.042795:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878530.042798:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878530.042800:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042803:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878530.042806:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878530.042814:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290072832/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878530.042827:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042831:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878530.042836:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042839:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878530.042841:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042845:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878530.042863:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878530.042866:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878530.042871:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878530.042873:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878530.042878:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290072832:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878530.042883:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042885:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878530.042887:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878530.042889:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878530.042892:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878530.042894:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878530.042896:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878530.042899:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878530.042929:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878530.042932:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878530.042934:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042936:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878530.042937:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878530.042939:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878530.042941:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878530.042943:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878530.042944:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878530.042946:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878530.042948:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878530.042950:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878530.042951:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042953:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878530.042954:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878530.042956:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878530.042957:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878530.042958:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878530.042960:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878530.042961:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878530.042962:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878530.042964:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878530.042965:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042967:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.042971:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878530.042973:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.042975:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878530.042977:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878530.104978:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878530.104991:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878530.104993:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878530.104998:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878530.105001:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878530.105004:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.105006:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878530.105008:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878530.105010:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878530.809018:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878530.809027:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878530.809034:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878530.809038:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878530.809042:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878530.809046:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878530.809050:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878530.809063:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878530.809065:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878530.809068:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878530.809154:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878530.809157:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878530.809162:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878531.129006:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878531.129014:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878531.129016:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878531.129021:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878531.129026:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878531.129029:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878531.129032:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878531.129035:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878531.129038:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878531.833014:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878531.833021:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878531.833026:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878531.833028:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878531.833107:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878531.833135:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878531.833141:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878532.153024:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878532.153033:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878532.153036:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878532.153043:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878532.153048:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878532.153053:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878532.153057:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878532.153060:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878532.153064:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878532.856994:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878532.856999:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878532.857003:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878532.857005:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878532.857085:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878532.857090:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878532.857095:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878533.177043:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878533.177053:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878533.177056:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878533.177063:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878533.177068:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878533.177072:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878533.177075:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878533.177078:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878533.177082:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878533.881065:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878533.881078:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878533.881082:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878533.881088:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878533.881123:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878533.881128:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878533.881133:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878533.881137:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878533.881144:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878533.881148:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878533.881194:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878533.881198:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878533.881201:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878534.201000:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878534.201012:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878534.201016:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878534.201026:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878534.201033:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878534.201039:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878534.201044:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878534.201048:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878534.201053:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878534.905082:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878534.905113:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878534.905121:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878534.905123:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878534.905173:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878534.905176:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878534.905180:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878535.161011:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878535.161027:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878535.161030:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878535.161034:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878535.161040:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878535.161058:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9474 00000100:00080000:2.0:1788878535.161073:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878535.161078:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878535.161080:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878535.161106:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878535.161110:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878535.161114:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878535.161117:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878535.161120:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878535.161137:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878535.161141:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878535.161143:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878535.161146:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878535.161150:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878535.161152:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878535.161154:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878535.161156:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878535.161171:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878535.161173:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878535.161177:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878535.161181:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878535.161186:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878535.161202:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878535.161215:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290072960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878535.161226:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878535.161230:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878535.161245:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878535.161248:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878535.161251:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878535.161252:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878535.161253:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878535.161256:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878535.161259:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9474 00000100:00080000:2.0:1788878535.161263:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878535.161265:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878535.161266:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878535.161269:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878535.161270:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788878535.161271:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878535.161275:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878535.161276:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788878535.161277:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878535.161281:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878535.161281:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878535.161285:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878535.161286:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878535.161289:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878535.161290:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:0.0:1788878535.161294:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788878535.161294:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878535.161296:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878535.161299:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878535.161302:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878535.161306:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788878535.161308:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290072960:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878535.161313:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.161314:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788878535.161318:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:0.0:1788878535.161321:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878535.161324:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878535.161326:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:2.0:1788878535.161326:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878535.161332:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878535.161333:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878535.161337:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878535.161345:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878535.161347:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878535.161348:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878535.161349:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878535.161351:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.161353:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878535.161355:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 00000100:00000001:2.0:1788878535.161358:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.161359:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1788878535.161367:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290073088/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878535.161377:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878535.161380:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878535.161383:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878535.161387:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878535.161391:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878535.161394:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878535.161396:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878535.161399:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000800:00000100:0.0:1788878535.161405:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788878535.161406:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9474 00000100:00080000:2.0:1788878535.161414:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878535.161416:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878535.161417:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878535.161421:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878535.161422:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878535.161423:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:2.0:1788878535.161424:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878535.161427:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878535.161429:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878535.161430:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878535.161433:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878535.161433:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878535.161437:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:2.0:1788878535.161440:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878535.161444:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878535.161444:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878535.161445:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878535.161450:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878535.161451:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304146985 02000000:00000001:2.0:1788878535.161454:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878535.161457:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878535.161459:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878535.161461:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878535.161462:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.161465:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1788878535.161467:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878535.161469:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878535.161470:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878535.161472:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878535.161474:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304146985 02000000:00000001:2.0:1788878535.161474:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878535.161479:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.161481:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00100000:2.0:1788878535.161485:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290073216/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878535.161509:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878535.161510:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878535.161516:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878535.161517:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.161523:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878535.161525:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878535.161530:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878535.161535:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878535/real 1788878535] req@ffff94e5045e6300 x1875765290072960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878535.161536:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878535.161540:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878535.161544:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878535.161553:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878535.161560:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9474 00000400:00000001:0.0:1788878535.161561:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878535.161566:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1788878535.161566:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878535.161572:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878535.161575:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878535.161575:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878535.161582:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000020:00000001:2.0:1788878535.161583:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878535.161586:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788878535.161587:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878535.161591:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1788878535.161593:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878535.161595:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788878535.161598:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878535.161604:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1788878535.161605:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290072960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878535.161607:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878535.161618:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878535.161627:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.161628:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878535.161630:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878535.161635:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788878535.161636:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878535.161644:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878535.161645:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.161651:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878535.161653:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878535.161654:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1788878535.161659:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878535.161660:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878535.161663:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878535.161669:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878535.161673:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878535.161675:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878535.161680:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290072960:192.168.203.129@tcp:250:kworker.0 00000008:00000001:2.0:1788878535.161680:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878535.161688:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.161691:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.161696:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878535.161698:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878535.161700:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878535.161701:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878535.161707:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878535.161709:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1788878535.161715:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:0.0:1788878535.161716:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878535.161720:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878535.161721:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878535.161726:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878535.161727:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878535.161731:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878535.161734:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878535.161736:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.161737:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.161745:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878535.161751:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290073344/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878535.161754:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878535.161765:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878535.161772:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.161774:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878535.161779:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.161780:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878535.161783:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878535.161787:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878535.161789:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878535.161791:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878535.161793:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878535.161795:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878535.161798:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878535.161802:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290073088:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878535.161806:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878535.161808:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878535.161810:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878535.161812:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878535.161815:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.161822:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878535.161824:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000100:00000001:0.0:1788878535.161829:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878535.161842:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878535.161847:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878535.161849:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878535.161850:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.161852:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878535.161854:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.161858:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304146443 00000100:00000001:0.0:1788878535.161861:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304146985 00000100:00000001:0.0:1788878535.161863:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304146443 00000100:00000001:0.0:1788878535.161866:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304146985 00000100:00000001:0.0:1788878535.161868:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878535.161869:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.161871:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878535.161874:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878535/real 1788878535] req@ffff94e5045e7800 x1875765290073088/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878535.161884:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878535.161885:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878535.161887:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878535.161889:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878535.161890:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.161891:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878535.161893:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878535.161896:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290073088/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878535.161932:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.161934:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878535.161937:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.161938:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878535.161939:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.161941:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878535.161958:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878535.161961:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878535.161964:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878535.161967:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878535.161970:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290073088:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878535.161974:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878535.161975:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878535.161977:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878535.161979:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878535.161981:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878535.161983:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878535.161985:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290073216:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878535.161988:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878535.161989:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878535.161991:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878535.161993:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878535.161996:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.162001:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878535.162002:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:0.0:1788878535.162006:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878535.162016:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878535.162020:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878535.162022:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878535.162024:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.162025:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878535.162028:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.162031:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304146442 00000100:00000001:0.0:1788878535.162033:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304146985 00000100:00000001:0.0:1788878535.162035:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304146442 00000100:00000001:0.0:1788878535.162038:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304146985 00000100:00000001:0.0:1788878535.162040:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878535.162041:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.162043:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878535.162046:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878535/real 1788878535] req@ffff94e5045e7100 x1875765290073216/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878535.162053:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878535.162054:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878535.162056:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878535.162057:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878535.162058:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.162060:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878535.162061:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878535.162065:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290073216/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878535.162071:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.162072:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878535.162075:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.162076:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878535.162078:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.162079:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878535.162116:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878535.162119:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878535.162122:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878535.162125:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878535.162128:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290073216:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878535.162131:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878535.162132:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878535.162149:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878535.162151:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878535.162153:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878535.162154:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878535.162156:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290073344:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878535.162159:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878535.162160:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878535.162161:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878535.162162:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878535.162164:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.162168:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878535.162169:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:0.0:1788878535.162172:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878535.162180:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878535.162183:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878535.162184:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878535.162186:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.162187:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878535.162188:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.162191:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304146442 00000100:00000001:0.0:1788878535.162193:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304146985 00000100:00000001:0.0:1788878535.162195:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304146442 00000100:00000001:0.0:1788878535.162197:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304146985 00000100:00000001:0.0:1788878535.162199:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878535.162200:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.162201:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878535.162204:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878535/real 1788878535] req@ffff94e5045e4a80 x1875765290073344/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878535.162208:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878535.162209:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878535.162211:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878535.162212:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878535.162213:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.162214:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878535.162215:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878535.162217:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290073344/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878535.162222:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.162223:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878535.162225:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.162226:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878535.162227:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.162228:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878535.162240:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878535.162241:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878535.162243:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878535.162245:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878535.162247:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290073344:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878535.162249:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.162250:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878535.162252:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878535.162253:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878535.162254:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878535.162256:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878535.162256:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878535.162258:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878535.162259:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878535.162261:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878535.162262:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.162263:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878535.162264:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878535.162265:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878535.162266:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878535.162267:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878535.162268:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878535.162269:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878535.162270:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878535.162271:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878535.162271:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.162272:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878535.162273:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878535.162274:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878535.162275:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878535.162275:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878535.162276:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878535.162277:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878535.162278:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878535.162279:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878535.162279:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.162280:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.162282:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878535.162283:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.162284:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878535.162285:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878535.225041:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878535.225066:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878535.225068:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878535.225073:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878535.225077:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878535.225081:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.225084:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878535.225086:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878535.225102:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878535.929031:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878535.929041:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878535.929048:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878535.929050:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878535.929142:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878535.929146:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878535.929150:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878536.249055:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878536.249068:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878536.249072:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878536.249109:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878536.249118:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878536.249126:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878536.249149:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878536.249154:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878536.249160:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878536.953047:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878536.953056:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878536.953063:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878536.953065:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878536.953187:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878536.953196:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878536.953204:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878537.273004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878537.273013:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878537.273016:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878537.273022:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878537.273027:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878537.273031:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878537.273035:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878537.273038:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878537.273042:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878537.977028:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878537.977035:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878537.977040:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878537.977042:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878537.977099:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878537.977130:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878537.977133:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878538.296973:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878538.296980:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878538.296982:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878538.296986:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878538.296989:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878538.296991:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878538.296993:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878538.296995:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878538.296997:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878539.001050:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878539.001094:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878539.001102:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878539.001104:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878539.001142:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878539.001145:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878539.001148:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878539.321094:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878539.321103:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878539.321105:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878539.321112:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878539.321116:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878539.321121:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878539.321124:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878539.321127:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878539.321131:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878540.025012:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878540.025020:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878540.025026:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878540.025028:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878540.025130:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878540.025136:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878540.025141:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878540.281120:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878540.281131:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878540.281135:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878540.281140:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878540.281145:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878540.281154:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9479 00000100:00080000:0.0:1788878540.281168:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878540.281172:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878540.281175:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878540.281186:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878540.281190:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878540.281195:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878540.281198:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878540.281201:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878540.281203:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878540.281208:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878540.281211:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878540.281215:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878540.281219:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878540.281222:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.281225:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878540.281228:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878540.281243:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878540.281245:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.281249:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878540.281252:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878540.281258:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.281273:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878540.281285:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290073472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878540.281299:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878540.281305:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.281330:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878540.281335:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878540.281339:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878540.281341:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878540.281344:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878540.281348:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878540.281351:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9479 00000100:00080000:0.0:1788878540.281355:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878540.281358:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878540.281361:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878540.281365:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878540.281367:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878540.281370:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788878540.281373:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878540.281376:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878540.281379:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878540.281383:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878540.281386:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878540.281388:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878540.281395:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878540.281399:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788878540.281400:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.281407:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878540.281408:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00100000:0.0:1788878540.281413:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788878540.281414:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878540.281422:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:0.0:1788878540.281423:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.281427:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.281429:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878540.281440:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878540.281441:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878540.281448:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878540.281449:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878540.281452:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878540.281459:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878540.281459:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878540.281466:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878540.281475:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290073472:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878540.281477:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.281489:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1788878540.281496:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290073600/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788878540.281498:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878540.281505:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878540.281513:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788878540.281520:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788878540.281523:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.281525:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.281536:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878540.281541:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878540.281545:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878540.281547:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878540.281549:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878540.281551:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878540.281554:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788878540.281557:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00080000:0.0:1788878540.281562:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9479 00000100:00000001:3.0:1788878540.281567:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788878540.281569:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878540.281574:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878540.281576:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878540.281580:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878540.281581:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878540.281584:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878540.281588:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878540.281591:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878540.281593:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878540.281610:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878540.281614:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788878540.281621:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:0.0:1788878540.281622:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878540.281630:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878540.281635:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.281638:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878540.281643:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788878540.281644:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00000001:0.0:1788878540.281648:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.281653:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.281654:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878540.281661:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878540.281662:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788878540.281667:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:0.0:1788878540.281668:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.281675:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878540.281678:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788878540.281680:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878540.281681:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.281695:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.281697:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304152105 00000100:00100000:0.0:1788878540.281706:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290073728/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878540.281718:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878540.281720:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878540.281722:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.281732:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878540.281736:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788878540.281740:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304152105 00000100:00080000:0.0:1788878540.281741:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878540.281746:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878540.281753:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878540.281757:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788878540.281759:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:0.0:1788878540.281761:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9479 00000100:00080000:0.0:1788878540.281771:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878540.281774:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878540.281775:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878540.281778:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878540.281780:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878540.281782:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788878540.281785:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878540.281788:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878540.281789:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878540.281793:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878540.281799:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878540.281803:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878540.281807:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788878540.281810:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878540.281815:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878540.281820:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878540.281826:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878540.281826:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878540/real 1788878540] req@ffff94e508978e00 x1875765290073472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878540.281833:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878540.281840:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878540.281848:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.281857:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788878540.281862:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1788878540.281865:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000400:00000001:3.0:1788878540.281868:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1788878540.281870:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.281877:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788878540.281879:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:0.0:1788878540.281880:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878540.281888:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.281889:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878540.281895:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.281896:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878540.281951:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290073856/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878540.281963:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878540.281967:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.281972:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.281978:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878540.281986:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878540.282001:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290073472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878540.282015:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282022:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878540.282028:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282032:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878540.282034:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282038:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878540.282042:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878540.282049:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878540.282075:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878540.282081:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878540.282089:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290073472:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878540.282099:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282103:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878540.282106:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878540.282111:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878540.282117:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878540.282121:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878540.282124:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878540.282132:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878540.282134:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878540.282139:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878540.282141:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282144:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282153:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878540.282157:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.282161:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878540.282164:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878540.282168:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878540.282171:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878540.282173:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878540.282178:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878540.282182:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878540.282186:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878540.282194:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290073600:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878540.282201:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878540.282205:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878540.282208:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878540.282212:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878540.282233:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.282247:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878540.282249:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:3.0:1788878540.282255:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878540.282278:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878540.282285:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878540.282289:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878540.282291:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.282293:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878540.282297:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.282302:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304151564 00000100:00000001:3.0:1788878540.282308:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304152105 00000100:00000001:3.0:1788878540.282313:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304151564 00000100:00000001:3.0:1788878540.282317:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304152105 00000100:00000001:3.0:1788878540.282321:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878540.282326:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.282329:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878540.282339:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878540/real 1788878540] req@ffff94e50897bb80 x1875765290073600/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878540.282357:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878540.282360:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878540.282365:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878540.282369:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878540.282372:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282376:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878540.282378:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878540.282387:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290073600/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878540.282403:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282406:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878540.282410:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282412:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878540.282414:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282416:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878540.282432:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878540.282436:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878540.282440:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878540.282443:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878540.282448:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290073600:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878540.282453:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878540.282455:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878540.282457:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878540.282460:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878540.282462:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878540.282465:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878540.282469:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290073728:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878540.282473:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878540.282476:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878540.282479:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878540.282481:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878540.282485:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.282493:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878540.282495:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:3.0:1788878540.282500:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878540.282518:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878540.282524:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878540.282527:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878540.282529:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.282531:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878540.282534:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.282538:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304151562 00000100:00000001:3.0:1788878540.282541:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304152105 00000100:00000001:3.0:1788878540.282545:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304151562 00000100:00000001:3.0:1788878540.282547:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304152105 00000100:00000001:3.0:1788878540.282550:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878540.282553:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.282555:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878540.282560:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878540/real 1788878540] req@ffff94e50897ad80 x1875765290073728/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878540.282570:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878540.282571:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878540.282574:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878540.282576:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878540.282578:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282580:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878540.282582:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878540.282586:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290073728/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878540.282595:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282597:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878540.282601:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282603:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878540.282605:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282607:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878540.282621:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878540.282624:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878540.282628:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878540.282631:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878540.282635:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290073728:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878540.282639:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878540.282640:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878540.282642:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878540.282644:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878540.282646:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878540.282649:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878540.282652:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290073856:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878540.282656:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878540.282657:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878540.282659:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878540.282661:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878540.282665:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.282670:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878540.282672:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:3.0:1788878540.282676:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878540.282693:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878540.282701:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878540.282705:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878540.282708:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.282711:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878540.282717:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.282723:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304151562 00000100:00000001:3.0:1788878540.282729:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304152105 00000100:00000001:3.0:1788878540.282734:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304151562 00000100:00000001:3.0:1788878540.282739:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304152105 00000100:00000001:3.0:1788878540.282744:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878540.282748:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.282752:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878540.282761:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878540/real 1788878540] req@ffff94e508979c00 x1875765290073856/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878540.282778:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878540.282781:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878540.282787:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878540.282790:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878540.282792:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282794:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878540.282796:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878540.282801:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290073856/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878540.282811:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282813:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878540.282816:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282818:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878540.282820:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282822:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878540.282836:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878540.282839:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878540.282842:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878540.282845:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878540.282849:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290073856:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878540.282853:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282856:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878540.282859:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878540.282861:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878540.282864:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878540.282866:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878540.282868:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878540.282871:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878540.282873:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878540.282877:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878540.282879:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282881:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878540.282883:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878540.282885:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878540.282887:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878540.282888:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878540.282889:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878540.282891:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878540.282893:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878540.282897:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878540.282899:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282943:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878540.282946:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878540.282949:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878540.282953:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878540.282956:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878540.282958:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878540.282961:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878540.282964:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878540.282968:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878540.282971:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282975:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878540.282982:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878540.282985:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878540.282989:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878540.282992:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878540.345038:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878540.345048:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878540.345073:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878540.345082:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878540.345088:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878540.345094:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878540.345098:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878540.345102:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878540.345123:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878541.049154:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878541.049161:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878541.049166:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878541.049170:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878541.049172:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878541.049175:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878541.049178:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878541.049180:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878541.049181:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878541.049184:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878541.049259:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878541.049263:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878541.049267:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878541.368998:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878541.369006:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878541.369010:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878541.369018:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878541.369023:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878541.369028:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878541.369032:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878541.369035:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878541.369039:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878542.073114:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878542.073122:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878542.073130:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878542.073133:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878542.073181:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878542.073186:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878542.073192:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878542.393068:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878542.393076:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878542.393079:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878542.393086:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878542.393090:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878542.393095:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878542.393098:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878542.393101:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878542.393104:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878543.097013:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878543.097024:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878543.097028:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878543.097046:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878543.097052:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878543.097055:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878543.097059:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878543.097062:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878543.097067:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878543.097068:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878543.097114:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878543.097117:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878543.097120:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878543.416995:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878543.417001:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878543.417003:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878543.417007:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878543.417011:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878543.417014:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878543.417016:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878543.417018:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878543.417021:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878544.120969:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878544.120974:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878544.120978:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878544.120979:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878544.121074:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878544.121080:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878544.121086:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878544.441114:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878544.441122:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878544.441124:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878544.441130:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878544.441134:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878544.441138:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878544.441141:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878544.441144:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878544.441146:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878545.145068:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878545.145077:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878545.145083:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878545.145085:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878545.145203:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878545.145215:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878545.145234:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788878545.401553:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878545.401568:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878545.401579:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878545.401585:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878545.401591:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878545.401628:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878545.401639:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878545.401657:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878545.401708:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290073984:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878545.401733:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878545.401747:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878545.401754:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878545.401775:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878545.401802:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.401841:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878545.401850:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286514304 : -117763423037312 : ffff94e511009880) 00000100:00000001:3.0:1788878545.401874:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878545.402039:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878545.402093:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878545.402114:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878545.402122:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.402126:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878545.402137:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.402147:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304157226 00000100:00000001:3.0:1788878545.402164:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878545.402174:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304157226 00000100:00000001:3.0:1788878545.402179:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878545.402194:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878545.402208:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.402211:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878545.402221:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878545/real 1788878545] req@ffff94e511009880 x1875765290073984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878545.402237:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878545.402241:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878545.402248:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878545.402256:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878545.402259:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.402264:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878545.402267:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878545.402272:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009880 x1875765290073984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878545.402281:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.402285:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878545.402288:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.402290:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878545.402291:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.402294:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878545.402296:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878545.402300:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878545.402305:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878545.402307:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878545.402311:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290073984:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878545.402317:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878545.402318:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878545.402320:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878545.402323:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878545.402326:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878545.402328:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878545.402331:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290074112:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878545.402335:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878545.402338:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878545.402340:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878545.402342:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878545.402345:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.402352:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878545.402355:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508928 : -117763423042688 : ffff94e511008380) 00000100:00000001:3.0:1788878545.402360:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878545.402377:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878545.402383:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878545.402385:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878545.402387:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.402389:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878545.402392:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.402397:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304156684 00000100:00000001:3.0:1788878545.402401:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304157226 00000100:00000001:3.0:1788878545.402404:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304156684 00000100:00000001:3.0:1788878545.402406:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304157226 00000100:00000001:3.0:1788878545.402550:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878545.402555:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.402558:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878545.402567:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878545/real 1788878545] req@ffff94e511008380 x1875765290074112/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878545.402583:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878545.402586:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878545.402591:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878545.402594:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878545.402596:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.402600:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878545.402602:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878545.402608:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008380 x1875765290074112/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878545.402636:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.402640:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878545.402644:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.402646:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878545.402649:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.402651:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878545.402677:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878545.402682:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878545.402687:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878545.402691:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878545.402697:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290074112:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878545.402703:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.402707:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878545.402710:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878545.402714:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878545.402720:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878545.402725:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878545.402729:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878545.402735:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878545.402738:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878545.402743:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878545.402745:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.402749:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878545.402750:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878545.402752:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878545.402755:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878545.402757:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878545.402759:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878545.402761:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878545.402764:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878545.402766:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878545.402768:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.402772:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.402784:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878545.402787:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.402792:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878545.402795:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878545.402797:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878545.402799:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878545.402802:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878545.402805:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878545.402807:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878545.402810:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878545.402815:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290074240:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878545.402819:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878545.402822:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878545.402824:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878545.402826:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878545.402830:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.402839:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878545.402840:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286512512 : -117763423039104 : ffff94e511009180) 00000100:00000001:3.0:1788878545.402846:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878545.402872:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878545.402877:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878545.402880:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878545.402882:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.402883:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878545.402886:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.402908:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304156682 00000100:00000001:3.0:1788878545.402911:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304157226 00000100:00000001:3.0:1788878545.402914:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304156682 00000100:00000001:3.0:1788878545.402916:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304157226 00000100:00000001:3.0:1788878545.402919:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878545.402921:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.402923:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878545.402928:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878545/real 1788878545] req@ffff94e511009180 x1875765290074240/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878545.402969:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878545.402971:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878545.402974:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878545.402976:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878545.402978:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.402981:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878545.402983:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878545.402990:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009180 x1875765290074240/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878545.403014:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.403034:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878545.403055:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.403058:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878545.403060:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.403064:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878545.403086:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878545.403092:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878545.403098:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878545.403102:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878545.403107:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290074240:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878545.403113:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878545.403115:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878545.403116:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878545.403120:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878545.403122:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878545.403124:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878545.403128:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290074368:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878545.403131:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878545.403134:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878545.403136:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878545.403138:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878545.403141:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.403148:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878545.403150:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286516096 : -117763423035520 : ffff94e511009f80) 00000100:00000001:3.0:1788878545.403154:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878545.403170:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878545.403176:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878545.403178:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878545.403180:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.403182:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878545.403185:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.403189:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304156683 00000100:00000001:3.0:1788878545.403192:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304157226 00000100:00000001:3.0:1788878545.403195:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304156683 00000100:00000001:3.0:1788878545.403197:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304157226 00000100:00000001:3.0:1788878545.403200:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878545.403202:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.403203:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878545.403208:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878545/real 1788878545] req@ffff94e511009f80 x1875765290074368/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878545.403215:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878545.403216:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878545.403219:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878545.403221:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878545.403222:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.403224:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878545.403226:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878545.403230:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009f80 x1875765290074368/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878545.403237:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.403240:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878545.403242:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.403244:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878545.403245:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.403247:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878545.403274:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878545.403277:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878545.403280:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878545.403282:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878545.403286:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290074368:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878545.403290:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.403292:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878545.403293:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878545.403295:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878545.403297:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878545.403299:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878545.403300:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878545.403302:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878545.403303:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878545.403305:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878545.403307:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.403309:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878545.403310:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878545.403311:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878545.403313:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878545.403314:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878545.403315:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878545.403317:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878545.403318:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878545.403319:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878545.403321:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.403322:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878545.403326:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878545.403328:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878545.403329:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878545.403331:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878546.168976:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878546.168982:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878546.168988:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878546.168989:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878546.169011:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878546.169026:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878546.169029:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878547.193029:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878547.193037:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878547.193043:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878547.193044:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878547.193070:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878547.193073:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878547.193076:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878548.217129:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878548.217140:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878548.217149:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878548.217151:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878548.217273:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878548.217280:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878548.217285:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878549.241006:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878549.241014:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878549.241019:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878549.241020:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878549.241093:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878549.241098:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878549.241101:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878550.265091:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878550.265101:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878550.265109:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878550.265112:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878550.265143:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878550.265148:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878550.265152:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788878550.521289:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878550.521296:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878550.521301:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878550.521308:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878550.521313:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878550.521320:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878550.521324:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878550.521329:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878550.521337:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290074496:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878550.521349:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878550.521354:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878550.521359:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878550.521367:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878550.521445:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878550.521468:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878550.521473:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286517888 : -117763423033728 : ffff94e51100a680) 00000100:00000001:3.0:1788878550.521482:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878550.521530:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878550.521537:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304162345 00000100:00000001:3.0:1788878550.521551:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878550.521562:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304162345 00000100:00000001:3.0:1788878550.521572:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878550.521579:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878550.521585:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878550.521592:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878550.521597:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878550.521720:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878550.521724:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788878550.521728:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878550.521730:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878550.521732:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878550.521734:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878550.521735:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878550.521737:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878550.521740:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878550.521743:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878550.521750:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290074624:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878550.521756:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878550.521759:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878550.521762:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878550.521766:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878550.521770:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878550.521781:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878550.521784:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:3.0:1788878550.521791:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878550.521814:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878550.521821:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304161803 00000100:00000001:3.0:1788878550.521826:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304162345 00000100:00000001:3.0:1788878550.521830:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304161803 00000100:00000001:3.0:1788878550.521833:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304162345 00000100:00000001:3.0:1788878550.521835:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878550.521838:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878550.521842:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878550.521844:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878550.521846:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878550.521849:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878550.521852:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878550.521855:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878550.521860:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290074752:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878550.521865:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878550.521868:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878550.521870:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878550.521873:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878550.521877:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878550.521885:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878550.521887:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:3.0:1788878550.521893:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878550.521953:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878550.521960:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304161802 00000100:00000001:3.0:1788878550.521964:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304162345 00000100:00000001:3.0:1788878550.521967:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304161802 00000100:00000001:3.0:1788878550.521971:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304162345 00000100:00000001:3.0:1788878550.521975:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878550.521978:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878550.522000:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878550.522003:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878550.522006:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878550.522011:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878550.522013:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878550.522016:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878550.522023:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290074880:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878550.522029:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878550.522033:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878550.522036:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878550.522039:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878550.522044:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878550.522054:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878550.522057:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:3.0:1788878550.522062:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878550.522081:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878550.522087:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304161802 00000100:00000001:3.0:1788878550.522091:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304162345 00000100:00000001:3.0:1788878550.522094:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304161802 00000100:00000001:3.0:1788878550.522097:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304162345 00000100:00000001:3.0:1788878550.522100:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878550.522103:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878550.522107:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878550.522110:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878550.522115:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878550.522117:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788878550.522120:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878550.522122:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878550.522124:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878550.522126:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878551.289065:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878551.289074:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878551.289080:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878551.289084:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878551.289088:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878551.289091:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878551.289095:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878551.289097:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878551.289100:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878551.289103:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878551.289129:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878551.289133:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878551.289136:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878552.313062:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878552.313070:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878552.313076:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878552.313078:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878552.313167:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878552.313173:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878552.313178:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878553.337068:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878553.337078:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878553.337081:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878553.337085:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878553.337091:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878553.337094:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878553.337096:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878553.337099:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878553.337103:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878553.337105:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878553.337208:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878553.337216:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878553.337222:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788878553.593189:0:2350:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1788878553.593228:0:2350:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788878553.593243:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788878553.593255:0:2350:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878553.593273:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878553.593275:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878553.593277:0:2350:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788878553.593285:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788878553.593289:0:2350:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878553.593291:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878553.593294:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878553.593295:0:2350:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788878553.593301:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788878553.593306:0:2350:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878553.593309:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878553.593310:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878553.593312:0:2350:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788878553.593316:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788878553.593321:0:2350:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878553.593323:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878553.593325:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878553.593327:0:2350:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878553.593510:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878553.593516:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878553.593519:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878553.593532:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878550/real 1788878553] req@ffff94e51100aa00 x1875765290074624/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878553.593549:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878553.593551:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878553.593558:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878553.593561:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878553.593564:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.593567:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878553.593569:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878553.593575:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290074624/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878553.593583:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.593587:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878553.593591:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.593593:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878553.593594:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.593598:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878553.593632:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878553.593637:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878553.593642:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878553.593645:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878553.593650:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290074624:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878553.593655:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878553.593659:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878550/real 1788878553] req@ffff94e51100b480 x1875765290074752/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878553.593669:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878553.593670:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878553.593673:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878553.593675:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878553.593676:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.593678:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878553.593680:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878553.593684:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290074752/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878553.593691:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.593694:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878553.593696:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.593698:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878553.593700:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.593702:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878553.593717:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878553.593720:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878553.593723:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878553.593726:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878553.593730:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290074752:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878553.593735:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878553.593740:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878550/real 1788878553] req@ffff94e51100b800 x1875765290074880/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878553.593750:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878553.593751:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878553.593754:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878553.593755:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878553.593757:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.593759:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878553.593761:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878553.593766:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290074880/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878553.593774:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.593777:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878553.593779:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.593781:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878553.593782:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.593784:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878553.593809:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878553.593813:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878553.593817:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878553.593932:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878553.593940:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290074880:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878553.593948:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878553.593954:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878550/real 1788878553] req@ffff94e51100a680 x1875765290074496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878553.593984:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878553.593990:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878553.593995:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878553.593997:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878553.594001:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.594004:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878553.594005:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878553.594010:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100a680 x1875765290074496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878553.594018:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.594021:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878553.594023:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.594025:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878553.594026:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.594028:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878553.594030:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878553.594033:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878553.594038:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878553.594040:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878553.594044:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290074496:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878553.594049:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.594052:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878553.594054:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878553.594057:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878553.594063:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878553.594066:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878553.594068:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878553.594072:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878553.594074:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878553.594078:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878553.594079:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.594082:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878553.594083:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878553.594084:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878553.594086:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878553.594088:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878553.594089:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878553.594091:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878553.594092:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878553.594095:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878553.594096:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.594098:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878553.594099:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878553.594100:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878553.594102:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878553.594103:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878553.594104:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878553.594106:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878553.594107:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878553.594109:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878553.594111:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.594113:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878553.594114:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878553.594116:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878553.594118:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878553.594119:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878553.594120:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878553.594122:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878553.594123:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878553.594125:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878553.594126:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.594129:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878553.594138:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878553.594140:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878553.594142:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878553.594144:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878554.361040:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878554.361048:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878554.361056:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878554.361058:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878554.361085:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878554.361088:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878554.361091:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878555.385017:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878555.385023:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878555.385027:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878555.385029:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878555.385066:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878555.385068:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878555.385071:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788878555.641346:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878555.641351:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878555.641354:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878555.641357:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878555.641360:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878555.641365:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878555.641369:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878555.641373:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878555.641379:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290075008:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878555.641385:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878555.641390:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878555.641392:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878555.641396:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878555.641401:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.641417:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878555.641419:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286523264 : -117763423028352 : ffff94e51100bb80) 00000100:00000001:3.0:1788878555.641424:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878555.641465:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878555.641479:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878555.641483:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878555.641486:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.641489:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878555.641496:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.641502:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304167465 00000100:00000001:3.0:1788878555.641513:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878555.641531:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304167465 00000100:00000001:3.0:1788878555.641537:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878555.641541:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878555.641546:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.641550:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878555.641557:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878555/real 1788878555] req@ffff94e51100bb80 x1875765290075008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878555.641572:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878555.641575:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878555.641580:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878555.641584:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878555.641586:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.641589:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878555.641592:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878555.641599:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100bb80 x1875765290075008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878555.641611:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.641615:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878555.641619:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.641628:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878555.641630:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.641633:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878555.641636:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878555.641640:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878555.641646:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878555.641650:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878555.641655:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290075008:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878555.641663:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.641666:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878555.641668:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878555.641671:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878555.641676:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878555.641678:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878555.641681:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878555.641686:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878555.641688:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878555.641692:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878555.641694:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.641698:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.641707:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878555.641709:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.641712:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878555.641714:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878555.641716:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878555.641718:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878555.641720:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878555.641732:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878555.641734:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878555.641737:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878555.641742:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290075136:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878555.641747:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878555.641749:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878555.641752:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878555.641754:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878555.641758:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.641767:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878555.641769:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286509824 : -117763423041792 : ffff94e511008700) 00000100:00000001:3.0:1788878555.641775:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878555.641794:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878555.641801:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878555.641804:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878555.641806:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.641808:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878555.641812:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.641817:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304166923 00000100:00000001:3.0:1788878555.641822:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304167465 00000100:00000001:3.0:1788878555.641826:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304166923 00000100:00000001:3.0:1788878555.641829:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304167465 00000100:00000001:3.0:1788878555.641832:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878555.641835:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.641838:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878555.641844:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878555/real 1788878555] req@ffff94e511008700 x1875765290075136/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878555.641854:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878555.641856:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878555.641859:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878555.641861:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878555.641863:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.641865:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878555.641867:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878555.641873:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008700 x1875765290075136/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878555.641883:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.641886:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878555.641889:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.641891:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878555.641892:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.641895:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878555.642673:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878555.642680:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878555.642704:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878555.642708:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878555.642728:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290075136:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878555.642736:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878555.642738:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878555.642740:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878555.642744:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878555.642747:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878555.642750:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878555.642755:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290075264:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878555.642760:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878555.642763:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878555.642766:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878555.642768:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878555.642774:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.642785:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878555.642788:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286511616 : -117763423040000 : ffff94e511008e00) 00000100:00000001:3.0:1788878555.642794:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878555.642820:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878555.642828:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878555.642859:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878555.642862:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.642865:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878555.642871:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.642877:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304166922 00000100:00000001:3.0:1788878555.642882:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304167465 00000100:00000001:3.0:1788878555.642886:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304166922 00000100:00000001:3.0:1788878555.642890:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304167465 00000100:00000001:3.0:1788878555.642893:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878555.642913:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.642916:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878555.642969:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878555/real 1788878555] req@ffff94e511008e00 x1875765290075264/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878555.642980:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878555.642981:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878555.642984:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878555.642986:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878555.642988:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.642990:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878555.642991:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878555.642995:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008e00 x1875765290075264/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878555.643002:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.643004:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878555.643006:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.643007:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878555.643009:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.643010:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878555.643020:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878555.643022:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878555.643025:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878555.643027:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878555.643029:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290075264:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878555.643033:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878555.643034:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878555.643036:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878555.643038:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878555.643040:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878555.643042:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878555.643044:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290075392:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878555.643047:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878555.643049:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878555.643050:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878555.643052:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878555.643055:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.643059:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878555.643061:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508032 : -117763423043584 : ffff94e511008000) 00000100:00000001:3.0:1788878555.643064:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878555.643077:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878555.643081:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878555.643083:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878555.643085:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.643086:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878555.643089:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.643092:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304166923 00000100:00000001:3.0:1788878555.643094:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304167465 00000100:00000001:3.0:1788878555.643096:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304166923 00000100:00000001:3.0:1788878555.643098:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304167465 00000100:00000001:3.0:1788878555.643100:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878555.643101:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.643103:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878555.643106:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878555/real 1788878555] req@ffff94e511008000 x1875765290075392/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878555.643112:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878555.643113:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878555.643115:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878555.643116:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878555.643117:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.643119:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878555.643120:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878555.643123:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008000 x1875765290075392/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878555.643129:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.643130:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878555.643132:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.643133:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878555.643134:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.643136:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878555.643145:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878555.643147:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878555.643150:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878555.643152:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878555.643154:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290075392:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878555.643158:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.643159:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878555.643161:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878555.643162:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878555.643164:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878555.643166:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878555.643167:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878555.643169:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878555.643171:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878555.643173:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878555.643174:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.643175:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878555.643176:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878555.643177:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878555.643179:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878555.643180:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878555.643181:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878555.643182:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878555.643183:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878555.643184:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878555.643185:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.643186:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878555.643187:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878555.643188:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878555.643189:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878555.643190:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878555.643191:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878555.643193:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878555.643194:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878555.643196:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878555.643197:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.643200:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878555.643206:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878555.643208:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878555.643211:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878555.643213:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878556.409018:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878556.409026:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878556.409031:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878556.409032:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878556.409162:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878556.409166:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878556.409169:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878557.433053:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878557.433063:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878557.433072:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878557.433075:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878557.433161:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878557.433167:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878557.433171:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878558.457028:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878558.457034:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878558.457038:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878558.457040:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878558.457060:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878558.457062:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878558.457064:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878559.481014:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878559.481021:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878559.481026:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878559.481027:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878559.481049:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878559.481052:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878559.481055:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878560.505030:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878560.505037:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878560.505042:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878560.505043:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878560.505128:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878560.505133:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878560.505137:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788878560.761296:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878560.761310:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878560.761322:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878560.761333:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878560.761348:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878560.761362:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878560.761375:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878560.761384:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878560.761413:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290075520:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878560.761435:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878560.761449:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878560.761466:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878560.761474:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878560.761493:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.761531:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878560.761536:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286519680 : -117763423031936 : ffff94e51100ad80) 00000100:00000001:3.0:1788878560.761545:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878560.761621:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878560.761645:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878560.761655:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878560.761660:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.761674:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878560.761683:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.761691:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304172585 00000100:00000001:3.0:1788878560.761708:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878560.761740:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304172585 00000100:00000001:3.0:1788878560.761750:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878560.761759:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878560.761763:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.761768:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878560.761782:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878560/real 1788878560] req@ffff94e51100ad80 x1875765290075520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878560.761800:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878560.761802:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878560.761807:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878560.761810:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878560.761813:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.761816:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878560.761819:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878560.761826:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100ad80 x1875765290075520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878560.761836:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.761840:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878560.761844:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.761846:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878560.761848:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.761852:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878560.761855:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878560.761859:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878560.761867:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878560.761872:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878560.761879:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290075520:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878560.761889:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878560.761892:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878560.761894:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878560.761898:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878560.761995:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878560.762000:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878560.762006:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290075648:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878560.762011:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878560.762015:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878560.762017:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878560.762020:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878560.762025:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.762034:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878560.762036:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508032 : -117763423043584 : ffff94e511008000) 00000100:00000001:3.0:1788878560.762043:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878560.762063:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878560.762070:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878560.762072:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878560.762075:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.762077:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878560.762080:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.762086:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304172044 00000100:00000001:3.0:1788878560.762090:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304172585 00000100:00000001:3.0:1788878560.762093:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304172044 00000100:00000001:3.0:1788878560.762096:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304172585 00000100:00000001:3.0:1788878560.762099:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878560.762101:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.762104:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878560.762109:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878560/real 1788878560] req@ffff94e511008000 x1875765290075648/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878560.762119:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878560.762121:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878560.762124:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878560.762126:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878560.762128:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762130:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878560.762132:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878560.762137:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008000 x1875765290075648/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878560.762154:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762157:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878560.762160:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762162:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878560.762164:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762166:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878560.762186:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878560.762189:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878560.762193:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878560.762196:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878560.762200:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290075648:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878560.762205:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762208:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878560.762211:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878560.762214:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878560.762218:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878560.762222:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878560.762224:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878560.762230:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878560.762233:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878560.762237:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878560.762239:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762241:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878560.762243:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878560.762245:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878560.762247:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878560.762248:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878560.762250:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878560.762252:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878560.762253:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878560.762255:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878560.762256:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762259:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762268:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878560.762272:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.762275:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878560.762277:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878560.762279:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878560.762281:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878560.762283:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878560.762286:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878560.762288:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878560.762290:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878560.762294:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290075776:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878560.762299:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878560.762301:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878560.762303:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878560.762305:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878560.762309:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.762316:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878560.762318:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286511616 : -117763423040000 : ffff94e511008e00) 00000100:00000001:3.0:1788878560.762323:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878560.762335:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878560.762340:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878560.762343:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878560.762345:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.762347:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878560.762350:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.762354:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304172042 00000100:00000001:3.0:1788878560.762358:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304172585 00000100:00000001:3.0:1788878560.762361:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304172042 00000100:00000001:3.0:1788878560.762375:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304172585 00000100:00000001:3.0:1788878560.762378:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878560.762380:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.762383:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878560.762387:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878560/real 1788878560] req@ffff94e511008e00 x1875765290075776/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878560.762405:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878560.762407:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878560.762410:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878560.762412:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878560.762413:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762415:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878560.762417:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878560.762421:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008e00 x1875765290075776/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878560.762429:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762432:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878560.762435:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762437:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878560.762439:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762441:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878560.762459:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878560.762462:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878560.762465:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878560.762468:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878560.762472:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290075776:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878560.762477:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878560.762479:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878560.762481:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878560.762483:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878560.762485:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878560.762487:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878560.762491:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290075904:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878560.762494:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878560.762497:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878560.762499:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878560.762501:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878560.762504:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.762509:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878560.762511:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286509824 : -117763423041792 : ffff94e511008700) 00000100:00000001:3.0:1788878560.762515:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878560.762526:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878560.762530:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878560.762533:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878560.762535:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.762537:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878560.762540:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.762543:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304172042 00000100:00000001:3.0:1788878560.762546:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304172585 00000100:00000001:3.0:1788878560.762549:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304172042 00000100:00000001:3.0:1788878560.762552:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304172585 00000100:00000001:3.0:1788878560.762566:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878560.762568:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.762570:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878560.762575:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878560/real 1788878560] req@ffff94e511008700 x1875765290075904/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878560.762583:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878560.762585:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878560.762587:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878560.762589:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878560.762590:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762592:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878560.762594:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878560.762598:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008700 x1875765290075904/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878560.762606:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762609:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878560.762612:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762615:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878560.762616:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762618:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878560.762635:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878560.762638:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878560.762642:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878560.762645:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878560.762649:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290075904:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878560.762652:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762655:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878560.762656:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878560.762658:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878560.762660:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878560.762661:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878560.762663:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878560.762665:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878560.762666:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878560.762669:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878560.762670:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762672:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878560.762673:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878560.762675:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878560.762677:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878560.762678:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878560.762680:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878560.762682:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878560.762684:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878560.762686:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878560.762687:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762690:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878560.762693:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878560.762694:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878560.762696:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878560.762698:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878561.529073:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878561.529080:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878561.529085:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878561.529088:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878561.529091:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878561.529094:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878561.529096:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878561.529098:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878561.529099:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878561.529101:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878561.529158:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878561.529162:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878561.529165:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878562.553050:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878562.553059:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878562.553076:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878562.553078:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878562.553108:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878562.553111:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878562.553115:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878563.577014:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878563.577023:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878563.577026:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878563.577030:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878563.577035:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878563.577038:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878563.577041:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878563.577045:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878563.577049:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878563.577051:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878563.577075:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878563.577078:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878563.577080:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878564.600971:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878564.600977:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878564.600981:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878564.600982:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878564.601040:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878564.601043:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878564.601046:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878565.624969:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878565.624982:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878565.624986:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878565.624987:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878565.625036:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878565.625038:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878565.625041:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:3.0:1788878565.881200:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878565.881207:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878565.881211:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878565.881216:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878565.881221:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878565.881227:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878565.881231:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878565.881236:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878565.881261:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290076032:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878565.881274:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878565.881281:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878565.881288:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878565.881295:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878565.881304:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.881331:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878565.881336:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286523264 : -117763423028352 : ffff94e51100bb80) 00000100:00000001:3.0:1788878565.881344:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878565.881408:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878565.881424:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878565.881430:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878565.881435:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.881437:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878565.881445:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.881452:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304177705 00000100:00000001:3.0:1788878565.881464:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878565.881484:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304177705 00000100:00000001:3.0:1788878565.881494:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878565.881510:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878565.881514:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.881519:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878565.881528:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878565/real 1788878565] req@ffff94e51100bb80 x1875765290076032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878565.881567:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878565.881574:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878565.881583:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878565.881590:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878565.881596:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.881605:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878565.881609:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878565.881621:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100bb80 x1875765290076032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878565.881640:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.881645:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878565.881652:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.881657:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878565.881663:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.881671:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878565.881675:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878565.881683:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878565.881694:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878565.881699:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878565.881709:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290076032:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878565.881717:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.881722:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878565.881725:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878565.881728:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878565.881734:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878565.881739:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878565.881744:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878565.881749:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878565.881751:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878565.881755:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878565.881756:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.881760:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.881768:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878565.881771:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.881775:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878565.881777:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878565.881780:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878565.881781:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878565.881783:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878565.881787:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878565.881789:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878565.881792:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878565.881798:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290076160:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878565.881803:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878565.881805:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878565.881806:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878565.881808:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878565.881811:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.881817:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878565.881818:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286517888 : -117763423033728 : ffff94e51100a680) 00000100:00000001:3.0:1788878565.881822:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878565.881836:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878565.881840:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878565.881842:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878565.881844:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.881845:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878565.881848:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.881852:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304177163 00000100:00000001:3.0:1788878565.881854:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304177705 00000100:00000001:3.0:1788878565.881856:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304177163 00000100:00000001:3.0:1788878565.881858:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304177705 00000100:00000001:3.0:1788878565.881860:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878565.881861:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.881863:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878565.881866:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878565/real 1788878565] req@ffff94e51100a680 x1875765290076160/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878565.881898:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878565.881899:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878565.881943:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878565.881946:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878565.881948:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.881950:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878565.881952:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878565.881959:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100a680 x1875765290076160/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878565.881969:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.881972:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878565.881975:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.881978:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878565.881991:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.881994:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878565.882009:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878565.882012:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878565.882015:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878565.882017:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878565.882020:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290076160:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878565.882024:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878565.882025:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878565.882027:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878565.882029:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878565.882031:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878565.882032:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878565.882034:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290076288:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878565.882037:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878565.882039:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878565.882041:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878565.882042:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878565.882045:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.882051:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878565.882052:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:3.0:1788878565.882055:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878565.882068:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878565.882072:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878565.882074:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878565.882076:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.882077:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878565.882079:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.882082:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304177162 00000100:00000001:3.0:1788878565.882084:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304177705 00000100:00000001:3.0:1788878565.882086:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304177162 00000100:00000001:3.0:1788878565.882088:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304177705 00000100:00000001:3.0:1788878565.882090:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878565.882092:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.882095:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878565.882100:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878565/real 1788878565] req@ffff94e51100b800 x1875765290076288/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878565.882110:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878565.882112:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878565.882115:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878565.882117:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878565.882118:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.882121:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878565.882122:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878565.882128:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290076288/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878565.882138:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.882141:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878565.882144:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.882147:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878565.882148:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.882151:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878565.882163:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878565.882165:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878565.882169:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878565.882170:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878565.882173:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290076288:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878565.882176:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878565.882178:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878565.882179:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878565.882181:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878565.882182:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878565.882184:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878565.882186:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290076416:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878565.882189:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878565.882191:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878565.882193:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878565.882194:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878565.882197:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.882201:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878565.882203:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:3.0:1788878565.882206:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878565.882217:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878565.882221:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878565.882223:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878565.882224:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.882225:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878565.882228:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.882231:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304177162 00000100:00000001:3.0:1788878565.882233:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304177705 00000100:00000001:3.0:1788878565.882235:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304177162 00000100:00000001:3.0:1788878565.882237:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304177705 00000100:00000001:3.0:1788878565.882238:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878565.882240:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.882241:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878565.882245:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878565/real 1788878565] req@ffff94e51100b480 x1875765290076416/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878565.882251:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878565.882251:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878565.882253:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878565.882255:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878565.882256:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.882257:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878565.882259:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878565.882261:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290076416/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878565.882266:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.882268:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878565.882270:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.882271:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878565.882272:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.882274:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878565.882288:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878565.882291:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878565.882296:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878565.882298:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878565.882303:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290076416:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878565.882307:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.882310:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878565.882312:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878565.882314:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878565.882326:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878565.882328:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878565.882330:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878565.882333:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878565.882335:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878565.882338:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878565.882340:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.882342:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878565.882343:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878565.882345:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878565.882347:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878565.882349:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878565.882350:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878565.882351:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878565.882352:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878565.882353:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878565.882354:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.882356:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878565.882357:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878565.882357:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878565.882359:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878565.882359:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878565.882360:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878565.882361:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878565.882362:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878565.882363:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878565.882364:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.882365:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878565.882369:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878565.882370:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878565.882371:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878565.882372:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878566.649025:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878566.649032:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878566.649038:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878566.649040:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878566.649066:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878566.649070:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878566.649073:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878567.673016:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878567.673022:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878567.673028:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878567.673030:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878567.673049:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878567.673051:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878567.673053:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878568.697006:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878568.697013:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878568.697019:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878568.697020:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878568.697073:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878568.697080:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878568.697085:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878569.720964:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878569.720970:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878569.720973:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878569.720975:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878569.720990:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878569.720992:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878569.720994:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878570.744984:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878570.744991:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878570.744996:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878570.744997:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878570.745024:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878570.745027:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878570.745030:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878571.001065:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878571.001081:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878571.001084:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878571.001089:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878571.001095:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878571.001104:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9509 00000100:00080000:2.0:1788878571.001119:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878571.001123:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878571.001125:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878571.001137:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878571.001140:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878571.001145:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878571.001148:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878571.001151:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878571.001152:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878571.001158:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878571.001160:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878571.001175:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878571.001178:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878571.001181:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878571.001183:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878571.001185:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878571.001200:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878571.001201:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878571.001204:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878571.001207:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878571.001211:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878571.001223:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878571.001233:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290076544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878571.001243:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878571.001248:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878571.001265:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878571.001268:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878571.001271:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878571.001272:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878571.001273:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878571.001276:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878571.001279:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9509 00000100:00080000:2.0:1788878571.001282:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878571.001284:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878571.001286:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878571.001290:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878571.001292:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878571.001294:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878571.001297:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878571.001299:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878571.001301:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878571.001303:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878571.001304:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878571.001306:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788878571.001308:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788878571.001313:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878571.001315:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788878571.001316:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878571.001320:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878571.001320:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:2.0:1788878571.001325:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788878571.001325:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878571.001329:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788878571.001332:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878571.001337:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788878571.001338:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878571.001342:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878571.001346:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878571.001346:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878571.001353:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878571.001359:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290076544:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878571.001363:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878571.001367:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788878571.001369:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878571.001371:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878571.001373:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878571.001378:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878571.001381:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878571.001381:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878571.001399:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878571.001408:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290076672/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878571.001408:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878571.001411:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 00000100:00000001:3.0:1788878571.001421:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878571.001428:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878571.001432:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878571.001439:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878571.001443:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878571.001448:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878571.001450:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878571.001453:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878571.001458:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000800:00000100:3.0:1788878571.001459:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788878571.001467:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9509 00000100:00000001:3.0:1788878571.001472:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788878571.001473:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878571.001478:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878571.001479:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878571.001485:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878571.001485:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.001492:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878571.001498:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.001502:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304182825 00000020:00000001:2.0:1788878571.001504:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878571.001508:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788878571.001510:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1788878571.001511:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878571.001529:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878571.001533:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878571.001535:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878571.001540:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878571.001543:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304182825 00000100:00000001:2.0:1788878571.001544:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878571.001554:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788878571.001555:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878571.001563:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878571.001564:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878571.001569:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878571.001570:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878571.001577:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878571.001580:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878571.001594:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.001595:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878571.001597:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878571.001606:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878571.001608:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878571.001615:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1788878571.001615:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878571/real 1788878571] req@ffff94e5045e5180 x1875765290076544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788878571.001617:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878571.001620:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878571.001632:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878571.001641:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290076800/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788878571.001645:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878571.001649:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878571.001656:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878571.001676:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878571.001679:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.001680:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878571.001684:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.001686:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878571.001691:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788878571.001693:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1788878571.001700:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878571.001702:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878571.001704:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878571.001706:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878571.001711:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878571.001715:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9509 00000100:00080000:2.0:1788878571.001718:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878571.001720:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878571.001721:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878571.001724:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878571.001725:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878571.001728:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:3.0:1788878571.001732:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290076544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788878571.001733:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878571.001740:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878571.001744:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878571.001760:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878571.001772:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.001775:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878571.001776:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878571.001785:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878571.001786:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788878571.001790:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.001792:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878571.001794:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878571.001801:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788878571.001801:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:2.0:1788878571.001806:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.001806:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878571.001823:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.001831:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1788878571.001832:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788878571.001834:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788878571.001836:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878571.001838:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788878571.001840:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878571.001854:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878571.001857:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878571.001861:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878571.001865:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.001865:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878571.001874:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290076544:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1788878571.001876:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290076928/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878571.001887:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.001896:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878571.001897:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878571.001977:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788878571.001979:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878571.001985:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878571.001985:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878571.001995:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878571.002001:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878571.002004:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878571.002010:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878571.002013:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878571.002019:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878571.002021:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002025:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002032:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878571.002034:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.002036:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878571.002038:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878571.002039:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878571.002041:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878571.002043:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878571.002045:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878571.002047:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878571.002050:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878571.002057:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290076672:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878571.002064:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878571.002067:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878571.002069:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878571.002072:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878571.002076:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.002096:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878571.002098:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:3.0:1788878571.002105:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878571.002126:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878571.002131:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878571.002134:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878571.002136:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.002137:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878571.002140:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.002144:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304182284 00000100:00000001:3.0:1788878571.002147:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304182825 00000100:00000001:3.0:1788878571.002149:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304182284 00000100:00000001:3.0:1788878571.002151:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304182825 00000100:00000001:3.0:1788878571.002153:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878571.002155:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.002157:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878571.002161:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878571/real 1788878571] req@ffff94e5045e5880 x1875765290076672/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878571.002185:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878571.002186:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878571.002188:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878571.002189:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878571.002190:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002191:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878571.002192:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878571.002195:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290076672/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878571.002204:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002206:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878571.002208:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002209:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878571.002210:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002211:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878571.002220:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878571.002222:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878571.002224:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878571.002226:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878571.002228:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290076672:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878571.002231:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878571.002232:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878571.002233:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878571.002235:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878571.002236:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878571.002237:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878571.002239:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290076800:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878571.002242:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878571.002244:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878571.002245:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878571.002246:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878571.002248:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.002251:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878571.002253:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:3.0:1788878571.002255:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878571.002263:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878571.002265:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878571.002267:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878571.002268:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.002269:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878571.002271:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.002273:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304182282 00000100:00000001:3.0:1788878571.002274:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304182825 00000100:00000001:3.0:1788878571.002276:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304182282 00000100:00000001:3.0:1788878571.002277:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304182825 00000100:00000001:3.0:1788878571.002279:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878571.002280:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.002281:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878571.002283:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878571/real 1788878571] req@ffff94e5045e6a00 x1875765290076800/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878571.002288:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878571.002289:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878571.002290:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878571.002291:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878571.002292:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002293:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878571.002294:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878571.002297:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290076800/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878571.002301:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002302:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878571.002303:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002304:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878571.002305:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002306:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878571.002314:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878571.002316:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878571.002318:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878571.002319:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878571.002321:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290076800:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878571.002324:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878571.002325:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878571.002326:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878571.002328:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878571.002329:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878571.002330:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878571.002332:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290076928:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878571.002333:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878571.002335:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878571.002335:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878571.002336:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878571.002338:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.002341:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878571.002342:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:3.0:1788878571.002344:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878571.002350:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878571.002352:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878571.002353:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878571.002354:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.002355:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878571.002356:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.002358:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304182282 00000100:00000001:3.0:1788878571.002360:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304182825 00000100:00000001:3.0:1788878571.002361:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304182282 00000100:00000001:3.0:1788878571.002363:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304182825 00000100:00000001:3.0:1788878571.002364:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878571.002365:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.002366:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878571.002369:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878571/real 1788878571] req@ffff94e5045e6680 x1875765290076928/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878571.002378:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878571.002379:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878571.002380:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878571.002381:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878571.002382:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002383:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878571.002383:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878571.002386:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290076928/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878571.002390:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002391:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878571.002392:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002393:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878571.002394:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002395:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878571.002404:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878571.002405:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878571.002407:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878571.002409:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878571.002411:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290076928:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878571.002413:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002414:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878571.002415:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878571.002416:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878571.002417:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878571.002418:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878571.002419:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878571.002420:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878571.002421:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878571.002422:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878571.002423:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002425:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878571.002425:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878571.002426:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878571.002427:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878571.002428:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878571.002429:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878571.002430:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878571.002431:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878571.002432:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878571.002433:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002434:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878571.002434:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878571.002435:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878571.002436:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878571.002436:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878571.002437:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878571.002438:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878571.002439:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878571.002440:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878571.002440:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002441:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878571.002444:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878571.002444:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878571.002446:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878571.002447:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878571.768976:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878571.768982:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878571.768986:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878571.768989:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878571.768990:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878571.768993:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878571.768995:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878571.768996:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878571.769003:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878571.769005:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878571.769029:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878571.769031:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878571.769033:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878572.792981:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878572.792986:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878572.792996:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878572.792997:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878572.793041:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878572.793045:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878572.793049:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878573.817015:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878573.817025:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878573.817028:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878573.817033:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878573.817053:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878573.817056:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878573.817059:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878573.817062:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878573.817067:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878573.817069:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878573.817106:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878573.817110:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878573.817113:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878574.841003:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878574.841011:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878574.841015:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878574.841017:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878574.841066:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878574.841069:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878574.841072:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878575.864994:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878575.865000:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878575.865004:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878575.865005:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878575.865042:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878575.865044:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878575.865047:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878576.121041:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878576.121052:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878576.121054:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878576.121058:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878576.121062:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878576.121069:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9514 00000100:00080000:0.0:1788878576.121079:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878576.121082:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878576.121084:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878576.121093:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878576.121095:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878576.121099:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878576.121101:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878576.121103:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878576.121104:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878576.121108:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878576.121111:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878576.121114:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878576.121117:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878576.121120:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878576.121122:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878576.121124:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878576.121135:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878576.121137:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878576.121139:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878576.121142:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878576.121146:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878576.121158:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878576.121167:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979180 x1875765290077056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878576.121177:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878576.121181:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878576.121196:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878576.121199:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878576.121202:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878576.121204:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878576.121205:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878576.121209:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878576.121212:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9514 00000100:00080000:0.0:1788878576.121216:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878576.121218:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878576.121220:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878576.121223:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878576.121226:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788878576.121226:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788878576.121231:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878576.121233:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788878576.121234:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878576.121237:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878576.121238:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878576.121241:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878576.121242:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878576.121246:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878576.121249:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878576.121254:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878576.121255:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878576.121258:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878576.121261:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878576.121265:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878576.121266:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878576.121271:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878576.121273:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290077056:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878576.121274:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878576.121281:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788878576.121281:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878576.121286:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:0.0:1788878576.121287:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878576.121292:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878576.121293:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878576.121296:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878576.121298:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878576.121299:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878576.121302:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878576.121304:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878576.121304:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878576.121306:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878576.121315:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878576.121322:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290077184/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878576.121326:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878576.121330:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 00000100:00000001:0.0:1788878576.121332:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878576.121335:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788878576.121336:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878576.121341:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878576.121344:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878576.121347:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878576.121348:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878576.121350:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878576.121352:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878576.121354:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9514 00000100:00080000:0.0:1788878576.121358:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878576.121359:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878576.121361:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878576.121364:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878576.121365:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878576.121367:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878576.121369:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000800:00000100:3.0:1788878576.121376:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878576.121382:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878576.121385:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878576.121389:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:0.0:1788878576.121390:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878576.121413:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121415:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1788878576.121416:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788878576.121422:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878576.121423:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878576.121428:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:0.0:1788878576.121429:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878576.121434:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878576.121437:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788878576.121438:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788878576.121443:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304187945 00000008:00000001:0.0:1788878576.121446:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878576.121452:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121456:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1788878576.121459:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878576.121463:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121464:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304187945 00000100:00000001:0.0:1788878576.121468:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788878576.121472:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1788878576.121474:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788878576.121479:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878576.121480:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121484:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121488:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788878576.121492:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878576.121495:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878576/real 1788878576] req@ffff94e508979180 x1875765290077056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788878576.121506:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290077312/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788878576.121511:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878576.121516:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878576.121522:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878576.121523:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878576.121528:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878576.121529:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121535:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878576.121536:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878576.121544:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788878576.121544:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878576.121549:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788878576.121551:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878576.121555:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878576.121557:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1788878576.121560:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290077056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878576.121561:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878576.121570:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9514 00000100:00000001:3.0:1788878576.121579:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788878576.121582:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878576.121584:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878576.121591:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878576.121591:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878576.121597:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878576.121598:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878576.121601:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.121606:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000020:00000001:0.0:1788878576.121608:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788878576.121609:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1788878576.121614:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00080000:3.0:1788878576.121616:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:0.0:1788878576.121620:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00080000:3.0:1788878576.121623:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000020:00000001:0.0:1788878576.121629:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878576.121630:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1788878576.121639:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1788878576.121641:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290077056:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878576.121642:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878576.121653:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788878576.121654:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878576.121660:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878576.121661:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121664:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878576.121665:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878576.121669:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878576.121671:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878576.121674:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:0.0:1788878576.121677:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878576.121678:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:0.0:1788878576.121681:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000020:00000001:3.0:1788878576.121682:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000008:00100000:0.0:1788878576.121684:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878576.121689:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878576.121689:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878576.121694:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878576.121695:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121702:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 02000000:00000001:0.0:1788878576.121703:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788878576.121705:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788878576.121723:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121726:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878576.121727:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878576.121731:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878576.121733:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121736:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878576.121738:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121739:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121744:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788878576.121748:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290077440/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878576.121749:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878576.121753:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878576.121756:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878576.121759:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878576.121766:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788878576.121767:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788878576.121770:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878576.121774:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878576.121782:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878576.121786:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878576.121790:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290077184:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878576.121794:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878576.121796:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878576.121798:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878576.121799:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878576.121803:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121823:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878576.121824:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000100:00000001:3.0:1788878576.121828:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878576.121840:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878576.121844:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878576.121846:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878576.121847:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121849:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878576.121851:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121854:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304187403 00000100:00000001:3.0:1788878576.121857:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304187945 00000100:00000001:3.0:1788878576.121859:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304187403 00000100:00000001:3.0:1788878576.121861:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304187945 00000100:00000001:3.0:1788878576.121863:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878576.121864:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121866:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878576.121869:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878576/real 1788878576] req@ffff94e50897b800 x1875765290077184/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878576.121875:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878576.121876:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878576.121878:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878576.121879:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878576.121881:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.121882:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878576.121883:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878576.121886:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290077184/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878576.121892:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.121893:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878576.121895:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.121896:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878576.121897:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.121899:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878576.121930:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878576.121933:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878576.121936:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878576.121938:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878576.121941:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290077184:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878576.121944:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878576.121945:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878576.121947:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878576.121948:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878576.121950:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878576.121952:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878576.121954:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290077312:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878576.121956:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878576.121958:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878576.121959:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878576.121961:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878576.121963:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121967:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878576.121968:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 00000100:00000001:3.0:1788878576.121971:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878576.121980:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878576.121984:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878576.121985:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878576.121987:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121988:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878576.121990:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.121993:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304187402 00000100:00000001:3.0:1788878576.121995:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304187945 00000100:00000001:3.0:1788878576.121997:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304187402 00000100:00000001:3.0:1788878576.121999:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304187945 00000100:00000001:3.0:1788878576.122002:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878576.122003:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.122005:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878576.122008:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878576/real 1788878576] req@ffff94e508978380 x1875765290077312/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878576.122014:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878576.122015:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878576.122017:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878576.122018:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878576.122019:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.122021:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878576.122022:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878576.122025:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290077312/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878576.122030:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.122032:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878576.122033:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.122035:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878576.122035:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.122037:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878576.122045:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878576.122047:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878576.122049:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878576.122050:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878576.122053:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290077312:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878576.122055:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.122057:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878576.122058:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878576.122059:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878576.122061:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878576.122062:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878576.122063:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878576.122066:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878576.122067:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878576.122069:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878576.122070:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.122072:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878576.122073:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878576.122074:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878576.122075:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878576.122076:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878576.122077:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878576.122079:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878576.122079:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878576.122081:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878576.122082:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.122083:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.122086:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878576.122087:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.122088:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878576.122089:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878576.122090:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878576.122091:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878576.122092:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878576.122094:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878576.122095:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878576.122097:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878576.122099:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290077440:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878576.122102:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878576.122103:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878576.122104:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878576.122105:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878576.122107:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.122111:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878576.122112:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:3.0:1788878576.122114:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878576.122121:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878576.122124:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878576.122125:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878576.122127:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.122128:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878576.122130:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.122133:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304187402 00000100:00000001:3.0:1788878576.122135:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304187945 00000100:00000001:3.0:1788878576.122137:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304187402 00000100:00000001:3.0:1788878576.122139:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304187945 00000100:00000001:3.0:1788878576.122140:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878576.122142:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.122143:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878576.122146:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878576/real 1788878576] req@ffff94e508979c00 x1875765290077440/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878576.122151:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878576.122152:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878576.122154:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878576.122155:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878576.122156:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.122157:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878576.122158:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878576.122161:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290077440/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878576.122167:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.122168:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878576.122170:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.122171:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878576.122172:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.122173:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878576.122182:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878576.122184:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878576.122186:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878576.122187:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878576.122190:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290077440:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878576.122193:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.122194:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878576.122195:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878576.122197:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878576.122198:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878576.122199:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878576.122200:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878576.122201:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878576.122202:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878576.122203:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878576.122204:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.122206:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878576.122208:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878576.122209:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878576.122210:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878576.122212:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878576.889003:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878576.889009:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878576.889013:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878576.889014:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878576.889067:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878576.889070:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878576.889073:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878577.912972:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878577.912977:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878577.912981:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878577.912982:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878577.913017:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878577.913020:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878577.913023:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878578.937034:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878578.937040:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878578.937045:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878578.937046:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878578.937070:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878578.937072:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878578.937075:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878579.961000:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878579.961006:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878579.961010:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878579.961012:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878579.961049:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878579.961052:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878579.961054:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878580.984960:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878580.984967:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878580.984972:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878580.984973:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878580.985022:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878580.985026:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878580.985029:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878581.241078:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878581.241090:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878581.241093:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878581.241097:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878581.241103:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878581.241111:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9520 00000100:00080000:2.0:1788878581.241125:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878581.241128:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878581.241130:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878581.241139:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878581.241142:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878581.241145:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878581.241148:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878581.241150:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878581.241151:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878581.241157:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878581.241159:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878581.241163:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878581.241168:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878581.241173:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878581.241176:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878581.241179:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878581.241197:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878581.241200:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878581.241205:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878581.241208:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878581.241215:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878581.241235:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878581.241249:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290077568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878581.241260:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878581.241265:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878581.241285:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878581.241288:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878581.241297:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878581.241298:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878581.241300:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878581.241303:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878581.241305:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9520 00000100:00080000:2.0:1788878581.241310:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878581.241312:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878581.241314:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878581.241317:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878581.241319:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878581.241321:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878581.241323:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878581.241326:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878581.241328:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878581.241332:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878581.241333:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878581.241335:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.241339:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788878581.241340:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788878581.241345:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878581.241349:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788878581.241350:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878581.241352:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878581.241358:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878581.241360:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788878581.241362:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788878581.241365:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:2.0:1788878581.241368:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788878581.241369:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788878581.241377:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878581.241382:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878581.241382:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290077568:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878581.241390:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788878581.241391:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878581.241393:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878581.241396:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878581.241397:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878581.241398:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878581.241401:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878581.241403:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878581.241406:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788878581.241412:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878581.241413:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878581.241421:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290077696/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878581.241429:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878581.241431:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878581.241435:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878581.241437:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878581.241440:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878581.241442:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878581.241443:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878581.241444:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878581.241446:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788878581.241448:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00080000:2.0:1788878581.241452:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9520 00000100:00000001:3.0:1788878581.241455:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788878581.241457:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878581.241461:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878581.241462:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878581.241465:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878581.241467:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878581.241469:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878581.241471:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878581.241473:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878581.241474:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878581.241477:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878581.241479:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878581.241481:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878581.241483:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878581.241485:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878581.241487:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878581.241490:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878581.241492:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878581.241494:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878581.241498:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878581.241499:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878581.241500:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878581.241502:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878581.241505:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788878581.241506:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878581.241511:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878581.241517:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290077824/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878581.241523:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878581.241524:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878581.241528:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.241530:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878581.241531:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.241534:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878581.241536:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788878581.241539:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:2.0:1788878581.241543:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878581.241548:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878581.241549:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878581.241553:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878581.241556:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304193065 00000100:00000001:2.0:1788878581.241557:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878581.241563:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9520 00000100:00080000:2.0:1788878581.241566:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878581.241569:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878581.241570:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878581.241571:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1788878581.241573:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878581.241576:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878581.241578:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878581.241580:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878581.241582:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878581.241583:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878581.241585:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878581.241587:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878581.241589:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788878581.241589:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304193065 02000000:00000001:2.0:1788878581.241592:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878581.241596:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.241597:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878581.241598:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878581.241602:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788878581.241605:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1788878581.241606:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878581.241613:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.241613:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.241620:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1788878581.241624:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878581.241626:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878581.241630:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1788878581.241631:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878581/real 1788878581] req@ffff94e5045e4700 x1875765290077568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788878581.241633:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878581.241637:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878581.241645:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788878581.241649:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:2.0:1788878581.241654:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290077952/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788878581.241655:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878581.241660:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878581.241663:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878581.241667:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878581.241667:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878581.241674:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.241674:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878581.241680:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.241680:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878581.241695:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290077568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878581.241708:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.241713:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878581.241717:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.241720:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878581.241721:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.241725:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878581.241729:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878581.241733:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878581.241739:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878581.241743:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878581.241747:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290077568:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878581.241752:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.241754:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878581.241756:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878581.241759:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878581.241763:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878581.241765:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878581.241767:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878581.241786:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878581.241788:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878581.241791:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878581.241792:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.241795:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.241803:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878581.241805:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.241807:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878581.241809:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878581.241811:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878581.241812:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878581.241814:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878581.241817:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878581.241820:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878581.241822:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878581.241827:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290077696:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878581.241840:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878581.241842:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878581.241844:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878581.241847:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878581.241851:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.241859:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878581.241862:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00000001:3.0:1788878581.241868:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878581.241888:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878581.241894:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878581.241897:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878581.241899:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.241935:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878581.241941:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.241947:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304192524 00000100:00000001:3.0:1788878581.241951:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304193065 00000100:00000001:3.0:1788878581.241954:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304192524 00000100:00000001:3.0:1788878581.241957:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304193065 00000100:00000001:3.0:1788878581.241960:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878581.241962:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.241965:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878581.241972:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878581/real 1788878581] req@ffff94e5045e7480 x1875765290077696/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878581.241984:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878581.241986:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878581.241990:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878581.241992:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878581.241994:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.241997:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878581.241999:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878581.242004:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290077696/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878581.242037:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.242040:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878581.242043:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.242045:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878581.242047:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.242049:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878581.242070:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878581.242075:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878581.242079:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878581.242082:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878581.242087:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290077696:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878581.242092:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878581.242093:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878581.242095:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878581.242098:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878581.242101:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878581.242104:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878581.242108:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290077824:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878581.242112:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878581.242115:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878581.242117:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878581.242119:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878581.242123:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.242132:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878581.242133:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00000001:3.0:1788878581.242138:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878581.242154:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878581.242160:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878581.242163:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878581.242165:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.242167:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878581.242171:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.242175:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304192522 00000100:00000001:3.0:1788878581.242179:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304193065 00000100:00000001:3.0:1788878581.242182:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304192522 00000100:00000001:3.0:1788878581.242185:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304193065 00000100:00000001:3.0:1788878581.242188:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878581.242190:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.242194:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878581.242199:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878581/real 1788878581] req@ffff94e5045e7b80 x1875765290077824/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878581.242211:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878581.242212:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878581.242216:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878581.242218:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878581.242220:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.242223:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878581.242225:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878581.242231:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290077824/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878581.242241:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.242244:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878581.242247:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.242249:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878581.242251:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.242253:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878581.242271:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878581.242274:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878581.242279:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878581.242282:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878581.242286:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290077824:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878581.242291:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878581.242293:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878581.242295:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878581.242298:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878581.242300:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878581.242302:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878581.242306:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290077952:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878581.242314:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878581.242316:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878581.242318:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878581.242321:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878581.242325:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.242332:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878581.242333:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:3.0:1788878581.242338:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878581.242354:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878581.242360:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878581.242363:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878581.242365:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.242367:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878581.242370:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.242375:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304192522 00000100:00000001:3.0:1788878581.242379:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304193065 00000100:00000001:3.0:1788878581.242381:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304192522 00000100:00000001:3.0:1788878581.242384:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304193065 00000100:00000001:3.0:1788878581.242387:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878581.242390:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.242393:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878581.242399:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878581/real 1788878581] req@ffff94e5045e4e00 x1875765290077952/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878581.242410:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878581.242412:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878581.242416:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878581.242418:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878581.242421:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.242423:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878581.242425:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878581.242431:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290077952/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878581.242449:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.242452:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878581.242456:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.242458:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878581.242460:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.242463:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878581.242479:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878581.242483:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878581.242487:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878581.242490:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878581.242494:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290077952:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878581.242500:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.242502:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878581.242504:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878581.242506:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878581.242510:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878581.242512:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878581.242514:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878581.242517:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878581.242518:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878581.242521:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878581.242523:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.242525:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878581.242527:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878581.242529:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878581.242531:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878581.242533:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878581.242535:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878581.242537:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878581.242539:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878581.242541:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878581.242543:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.242545:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878581.242546:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878581.242548:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878581.242550:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878581.242552:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878581.242554:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878581.242556:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878581.242557:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878581.242560:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878581.242562:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.242564:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878581.242569:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878581.242571:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878581.242574:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878581.242576:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878582.009023:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878582.009030:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878582.009035:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878582.009038:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878582.009040:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878582.009044:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878582.009046:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878582.009048:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878582.009049:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878582.009051:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878582.009075:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878582.009077:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878582.009080:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878583.033005:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878583.033016:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878583.033020:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878583.033023:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878583.033030:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878583.033032:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878583.033034:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878583.033053:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878583.033056:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878583.033057:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878583.033078:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878583.033080:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878583.033083:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878584.057020:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878584.057030:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878584.057038:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878584.057040:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878584.057067:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878584.057070:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878584.057073:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878585.081067:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878585.081078:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878585.081086:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878585.081088:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878585.081224:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878585.081232:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878585.081238:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878586.105061:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878586.105068:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878586.105073:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878586.105074:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878586.105100:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878586.105103:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878586.105105:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878586.360981:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878586.360994:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878586.360997:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878586.361002:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878586.361008:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878586.361017:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9525 00000100:00080000:2.0:1788878586.361030:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878586.361033:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878586.361035:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878586.361045:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878586.361048:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878586.361052:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878586.361054:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878586.361056:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878586.361057:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878586.361062:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878586.361064:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878586.361067:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878586.361071:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878586.361074:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361076:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878586.361078:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878586.361093:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878586.361094:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361098:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878586.361101:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878586.361107:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361122:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878586.361131:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4000 x1875765290078080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878586.361142:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878586.361148:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361170:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878586.361175:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878586.361180:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878586.361182:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878586.361184:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878586.361189:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878586.361193:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9525 00000100:00080000:2.0:1788878586.361199:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878586.361218:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878586.361220:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878586.361227:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878586.361229:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878586.361233:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878586.361236:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878586.361239:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878586.361240:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878586.361244:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878586.361246:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878586.361247:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878586.361252:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788878586.361258:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788878586.361261:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878586.361265:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.361265:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878586.361271:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878586.361284:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361299:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788878586.361301:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:2.0:1788878586.361306:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788878586.361306:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1788878586.361321:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878586.361321:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361337:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878586.361344:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290078080:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878586.361346:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878586.361350:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361356:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788878586.361357:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878586.361359:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878586.361364:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878586.361364:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878586.361367:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361377:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878586.361378:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878586.361388:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878586.361390:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290078208/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878586.361404:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878586.361405:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878586.361407:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074548224 : -117763635003392 : ffff94e5045e4000) 00000100:00000001:2.0:1788878586.361414:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.361415:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878586.361426:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878586.361431:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878586.361435:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878586.361436:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878586.361438:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878586.361441:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878586.361445:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9525 00000100:00080000:2.0:1788878586.361450:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878586.361455:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000800:00000100:3.0:1788878586.361456:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878586.361458:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878586.361463:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878586.361467:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878586.361471:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878586.361472:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:2.0:1788878586.361482:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878586.361486:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878586.361488:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878586.361489:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361490:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878586.361494:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:2.0:1788878586.361499:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878586.361500:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361508:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878586.361512:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788878586.361512:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304198185 02000000:00000001:2.0:1788878586.361524:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878586.361528:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361530:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878586.361530:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00100000:2.0:1788878586.361537:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878586.361542:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.361542:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304198185 00000100:00000001:2.0:1788878586.361548:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.361552:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1788878586.361555:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788878586.361557:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878586.361558:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.361563:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361564:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878586.361567:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788878586.361567:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1788878586.361575:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878586.361582:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878586/real 1788878586] req@ffff94e5045e4000 x1875765290078080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878586.361585:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878586.361603:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290078336/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788878586.361605:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878586.361610:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878586.361616:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878586.361620:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878586.361626:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.361636:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878586.361649:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878586.361659:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4000 x1875765290078080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878586.361665:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878586.361673:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361682:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878586.361692:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788878586.361693:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.361702:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878586.361718:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878586.361722:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878586.361723:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878586.361728:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878586.361730:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878586.361733:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788878586.361748:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.361756:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1788878586.361761:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9525 00000100:00000001:3.0:1788878586.361761:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878586.361781:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878586.361791:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788878586.361792:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878586.361798:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878586.361800:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788878586.361801:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878586.361816:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:3.0:1788878586.361819:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290078080:192.168.203.129@tcp:250:kworker.0 00000020:00000001:2.0:1788878586.361820:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878586.361824:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878586.361830:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878586.361853:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878586.361854:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878586.361856:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878586.361860:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878586.361866:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.361867:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788878586.361868:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788878586.361871:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878586.361873:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878586.361875:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878586.361878:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361883:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788878586.361884:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000008:00100000:2.0:1788878586.361890:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000020:00000001:3.0:1788878586.361891:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878586.361895:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000008:00000001:2.0:1788878586.361897:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361899:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878586.361903:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878586.361905:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361907:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878586.361909:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878586.361912:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361917:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878586.361922:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290078464/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878586.361929:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878586.361931:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878586.361942:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878586.361944:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878586.361946:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878586.361948:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.361950:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.361956:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878586.361958:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.361960:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878586.361962:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878586.361963:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878586.361964:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878586.361966:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878586.361969:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878586.361971:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878586.361976:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878586.361978:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878586.361987:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290078208:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878586.361991:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878586.361995:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878586.361996:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878586.361998:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878586.362002:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.362012:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878586.362013:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074559872 : -117763634991744 : ffff94e5045e6d80) 00000100:00000001:3.0:1788878586.362018:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878586.362034:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878586.362038:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878586.362040:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878586.362041:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.362042:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878586.362044:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.362047:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304197644 00000100:00000001:3.0:1788878586.362049:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304198185 00000100:00000001:3.0:1788878586.362053:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304197644 00000100:00000001:3.0:1788878586.362055:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304198185 00000100:00000001:3.0:1788878586.362057:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878586.362059:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.362060:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878586.362064:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878586/real 1788878586] req@ffff94e5045e6d80 x1875765290078208/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878586.362071:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878586.362073:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878586.362076:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878586.362083:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878586.362085:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362086:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878586.362087:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878586.362090:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6d80 x1875765290078208/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878586.362095:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362098:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878586.362101:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362104:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878586.362105:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362107:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878586.362120:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878586.362122:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878586.362124:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878586.362126:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878586.362129:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290078208:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878586.362132:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878586.362134:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878586.362136:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878586.362138:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878586.362141:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878586.362142:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878586.362147:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290078336:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878586.362149:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878586.362150:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878586.362151:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878586.362152:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878586.362158:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.362162:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878586.362164:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:3.0:1788878586.362167:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878586.362180:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878586.362184:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878586.362185:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878586.362186:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.362188:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878586.362190:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.362192:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304197642 00000100:00000001:3.0:1788878586.362194:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304198185 00000100:00000001:3.0:1788878586.362195:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304197642 00000100:00000001:3.0:1788878586.362197:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304198185 00000100:00000001:3.0:1788878586.362199:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878586.362201:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.362202:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878586.362207:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878586/real 1788878586] req@ffff94e5045e4e00 x1875765290078336/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878586.362223:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878586.362225:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878586.362233:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878586.362236:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878586.362238:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362242:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878586.362244:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878586.362252:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290078336/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878586.362261:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362264:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878586.362269:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362273:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878586.362276:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362278:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878586.362384:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878586.362388:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878586.362391:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878586.362393:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878586.362396:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290078336:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878586.362400:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362402:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878586.362403:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878586.362405:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878586.362407:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878586.362408:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878586.362409:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878586.362412:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878586.362413:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878586.362414:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878586.362415:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362416:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878586.362417:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878586.362418:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878586.362419:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878586.362420:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878586.362420:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878586.362422:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878586.362422:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878586.362423:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878586.362424:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362426:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362428:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878586.362429:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.362431:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878586.362432:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878586.362433:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878586.362434:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878586.362435:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878586.362436:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878586.362438:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878586.362439:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878586.362441:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290078464:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878586.362443:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878586.362445:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878586.362446:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878586.362447:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878586.362450:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.362454:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878586.362455:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00000001:3.0:1788878586.362458:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878586.362467:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878586.362470:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878586.362472:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878586.362473:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.362474:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878586.362476:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.362479:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304197642 00000100:00000001:3.0:1788878586.362480:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304198185 00000100:00000001:3.0:1788878586.362482:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304197642 00000100:00000001:3.0:1788878586.362483:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304198185 00000100:00000001:3.0:1788878586.362485:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878586.362486:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.362487:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878586.362490:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878586/real 1788878586] req@ffff94e5045e7b80 x1875765290078464/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878586.362495:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878586.362496:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878586.362498:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878586.362499:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878586.362500:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362501:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878586.362502:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878586.362504:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290078464/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878586.362509:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362510:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878586.362511:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362512:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878586.362513:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362514:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878586.362523:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878586.362524:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878586.362526:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878586.362527:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878586.362529:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290078464:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878586.362531:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362532:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878586.362533:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878586.362534:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878586.362535:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878586.362536:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878586.362537:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878586.362538:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878586.362539:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878586.362541:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878586.362541:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362543:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878586.362545:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878586.362546:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878586.362547:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878586.362678:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878587.128992:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878587.128999:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878587.129004:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878587.129005:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878587.129027:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878587.129029:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878587.129032:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878588.153063:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878588.153072:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878588.153077:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878588.153078:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878588.153107:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878588.153109:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878588.153112:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878589.177040:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878589.177045:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878589.177049:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878589.177050:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878589.177126:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878589.177130:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878589.177134:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878590.201037:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878590.201044:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878590.201050:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878590.201052:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878590.201084:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878590.201088:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878590.201093:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878591.225082:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878591.225096:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878591.225108:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878591.225114:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878591.225118:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878591.225126:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878591.225132:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878591.225136:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878591.225141:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878591.225146:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878591.225239:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878591.225245:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878591.225251:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878591.480997:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878591.481007:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878591.481010:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878591.481014:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878591.481023:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878591.481031:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9530 00000100:00080000:2.0:1788878591.481042:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878591.481045:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878591.481047:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878591.481055:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878591.481058:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878591.481061:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878591.481064:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878591.481065:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878591.481067:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878591.481070:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878591.481072:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878591.481075:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878591.481078:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878591.481080:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878591.481082:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878591.481084:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878591.481097:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878591.481099:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878591.481102:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878591.481105:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878591.481110:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878591.481124:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878591.481133:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290078592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878591.481142:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878591.481147:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878591.481162:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878591.481164:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878591.481167:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878591.481168:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878591.481170:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878591.481172:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878591.481174:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9530 00000100:00080000:2.0:1788878591.481177:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878591.481179:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878591.481180:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878591.481182:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878591.481183:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878591.481185:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878591.481186:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878591.481188:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878591.481189:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878591.481193:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878591.481195:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878591.481196:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878591.481202:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878591.481204:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788878591.481205:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878591.481208:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878591.481211:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1788878591.481214:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788878591.481215:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878591.481218:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:2.0:1788878591.481219:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481222:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878591.481223:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878591.481230:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878591.481230:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878591.481231:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878591.481234:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878591.481235:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878591.481238:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878591.481240:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878591.481241:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878591.481247:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290078592:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878591.481251:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481252:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878591.481258:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878591.481262:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878591.481263:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290078720/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788878591.481265:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878591.481271:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878591.481279:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878591.481282:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878591.481286:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878591.481289:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878591.481294:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878591.481296:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878591.481298:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878591.481316:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878591.481320:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788878591.481322:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00080000:2.0:1788878591.481331:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9530 00000100:00000001:3.0:1788878591.481332:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788878591.481344:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878591.481348:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878591.481350:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878591.481375:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878591.481378:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878591.481382:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878591.481385:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878591.481388:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878591.481390:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878591.481396:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:3.0:1788878591.481405:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878591.481407:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878591.481415:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878591.481420:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878591.481424:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878591.481428:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878591.481432:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788878591.481435:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00000001:2.0:1788878591.481437:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878591.481444:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481450:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788878591.481453:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878591.481459:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481459:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878591.481472:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788878591.481472:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:2.0:1788878591.481480:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878591.481485:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481489:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878591.481494:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481502:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304203305 00000100:00100000:2.0:1788878591.481507:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290078848/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878591.481522:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878591.481526:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481527:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878591.481531:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878591.481538:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878591.481543:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878591.481545:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878591.481564:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878591.481566:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304203305 00000100:00000001:2.0:1788878591.481567:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878591.481574:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9530 00000100:00000001:3.0:1788878591.481574:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1788878591.481578:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878591.481581:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878591.481582:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878591.481586:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788878591.481587:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878591.481591:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788878591.481592:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:2.0:1788878591.481595:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878591.481598:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:3.0:1788878591.481602:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878591/real 1788878591] req@ffff94e5045e7480 x1875765290078592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788878591.481603:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878591.481610:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878591.481614:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878591.481623:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000400:00000001:3.0:1788878591.481625:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788878591.481626:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788878591.481629:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1788878591.481630:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878591.481635:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878591.481635:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:2.0:1788878591.481636:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878591.481640:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878591.481641:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000008:00100000:2.0:1788878591.481642:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788878591.481644:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1788878591.481646:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481649:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878591.481650:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481654:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878591.481657:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1788878591.481660:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290078592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788878591.481687:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878591.481722:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788878591.481729:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878591.481751:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878591.481755:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481755:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878591.481762:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.481764:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878591.481765:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481766:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.481772:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878591.481774:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:2.0:1788878591.481775:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290078976/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1788878591.481779:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788878591.481789:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00080000:3.0:1788878591.481816:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878591.481819:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481821:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788878591.481823:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878591.481828:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290078592:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878591.481834:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.481837:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878591.481838:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878591.481841:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878591.481845:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878591.481848:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878591.481850:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878591.481866:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878591.481868:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878591.481871:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878591.481872:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.481874:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.481882:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878591.481884:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481885:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878591.481887:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878591.481904:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878591.481905:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878591.481906:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878591.481908:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878591.481910:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878591.481911:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878591.481914:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290078720:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878591.481916:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878591.481917:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878591.481918:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878591.481919:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878591.481922:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481928:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878591.481929:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:3.0:1788878591.481933:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878591.481943:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878591.481947:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878591.481949:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878591.481972:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481973:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878591.481976:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481980:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304202764 00000100:00000001:3.0:1788878591.481982:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304203305 00000100:00000001:3.0:1788878591.481984:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304202764 00000100:00000001:3.0:1788878591.481986:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304203305 00000100:00000001:3.0:1788878591.481987:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878591.481989:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.481990:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878591.481994:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878591/real 1788878591] req@ffff94e5045e4700 x1875765290078720/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878591.482000:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878591.482001:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878591.482003:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878591.482004:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878591.482005:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482006:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878591.482007:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878591.482010:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290078720/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878591.482015:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482016:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878591.482018:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482019:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878591.482020:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482021:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878591.482035:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878591.482036:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878591.482038:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878591.482040:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878591.482042:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290078720:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878591.482044:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878591.482045:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878591.482046:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878591.482048:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878591.482050:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878591.482051:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878591.482053:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290078848:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878591.482058:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878591.482059:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878591.482060:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878591.482061:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878591.482064:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.482068:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878591.482069:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:3.0:1788878591.482071:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878591.482079:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878591.482082:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878591.482084:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878591.482085:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.482086:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878591.482088:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.482090:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304202762 00000100:00000001:3.0:1788878591.482092:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304203305 00000100:00000001:3.0:1788878591.482094:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304202762 00000100:00000001:3.0:1788878591.482095:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304203305 00000100:00000001:3.0:1788878591.482097:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878591.482098:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.482100:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878591.482102:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878591/real 1788878591] req@ffff94e5045e6680 x1875765290078848/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878591.482107:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878591.482107:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878591.482109:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878591.482110:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878591.482110:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482112:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878591.482112:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878591.482115:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290078848/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878591.482124:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482126:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878591.482128:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482128:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878591.482129:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482130:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878591.482142:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878591.482143:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878591.482145:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878591.482163:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878591.482167:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290078848:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878591.482172:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878591.482174:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878591.482175:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878591.482178:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878591.482181:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878591.482183:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878591.482186:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290078976:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878591.482190:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878591.482192:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878591.482194:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878591.482196:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878591.482199:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.482205:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878591.482206:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:3.0:1788878591.482211:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788878591.482223:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878591.482228:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878591.482231:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878591.482232:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.482235:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878591.482238:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.482242:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304202762 00000100:00000001:3.0:1788878591.482245:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304203305 00000100:00000001:3.0:1788878591.482248:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304202762 00000100:00000001:3.0:1788878591.482251:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304203305 00000100:00000001:3.0:1788878591.482253:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878591.482255:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.482257:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788878591.482262:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878591/real 1788878591] req@ffff94e5045e6a00 x1875765290078976/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788878591.482271:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788878591.482272:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878591.482275:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878591.482276:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878591.482278:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482280:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878591.482281:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788878591.482286:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290078976/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878591.482294:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482297:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878591.482300:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482302:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878591.482303:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482306:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878591.482321:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878591.482324:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788878591.482327:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878591.482330:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878591.482334:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290078976:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788878591.482339:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482341:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878591.482343:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878591.482345:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878591.482348:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878591.482349:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878591.482351:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878591.482354:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878591.482355:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878591.482358:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878591.482359:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482361:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878591.482363:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878591.482365:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878591.482367:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878591.482368:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878591.482370:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878591.482373:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878591.482374:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878591.482376:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878591.482378:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482379:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878591.482381:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878591.482382:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878591.482384:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788878591.482385:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788878591.482386:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878591.482388:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878591.482389:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878591.482391:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878591.482392:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482394:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878591.482398:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878591.482400:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878591.482402:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878591.482405:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878592.249004:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878592.249013:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878592.249018:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878592.249019:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878592.249044:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878592.249047:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878592.249050:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878593.273059:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878593.273075:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878593.273081:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878593.273087:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878593.273096:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878593.273101:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878593.273104:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878593.273107:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878593.273112:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878593.273114:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878593.273254:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878593.273263:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878593.273270:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878594.297040:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878594.297050:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878594.297058:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878594.297059:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878594.297090:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878594.297093:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878594.297097:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878595.321025:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878595.321032:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878595.321038:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878595.321039:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878595.321062:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878595.321064:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878595.321067:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878596.345036:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878596.345046:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878596.345053:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878596.345055:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878596.345086:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878596.345089:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878596.345093:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878596.601070:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878596.601080:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878596.601082:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878596.601086:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878596.601090:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878596.601098:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9535 00000100:00080000:3.0:1788878596.601108:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878596.601112:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878596.601114:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878596.601123:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878596.601126:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878596.601129:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878596.601132:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878596.601135:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878596.601136:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878596.601140:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878596.601142:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878596.601145:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878596.601148:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878596.601150:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601152:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878596.601153:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878596.601162:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878596.601163:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601166:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878596.601168:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878596.601172:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601182:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878596.601189:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290079104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878596.601197:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878596.601200:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601223:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878596.601225:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878596.601227:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878596.601228:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878596.601229:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878596.601231:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878596.601233:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9535 00000100:00080000:3.0:1788878596.601236:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878596.601238:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878596.601239:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878596.601241:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878596.601242:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878596.601243:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878596.601245:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878596.601246:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878596.601247:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878596.601250:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878596.601251:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878596.601252:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878596.601257:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878596.601258:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601260:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878596.601263:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878596.601265:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601266:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601270:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788878596.601270:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878596.601271:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601275:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878596.601276:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878596.601277:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788878596.601278:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601280:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878596.601284:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878596.601284:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878596.601289:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290079232/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878596.601294:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601300:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1788878596.601301:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878596.601304:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878596.601305:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878596.601308:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878596.601309:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878596.601311:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878596.601316:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290079104:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1788878596.601316:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878596.601317:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878596.601320:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878596.601326:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878596.601327:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788878596.601330:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9535 02000000:00000001:0.0:1788878596.601334:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1788878596.601334:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878596.601338:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788878596.601339:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601339:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878596.601345:0:177900:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:0.0:1788878596.601346:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1788878596.601347:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878596.601350:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878596.601352:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878596.601353:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788878596.601354:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601355:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878596.601361:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878596.601362:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878596.601363:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878596.601365:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878596.601367:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601368:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878596.601369:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878596.601370:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601372:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601375:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788878596.601376:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878596.601378:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 02000000:00000001:3.0:1788878596.601379:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601383:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878596.601384:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788878596.601385:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878596.601388:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601392:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878596.601396:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290079360/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878596.601401:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878596.601402:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601404:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878596.601405:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878596.601408:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878596.601409:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878596.601410:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878596.601412:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878596.601414:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9535 00000800:00000100:0.0:1788878596.601418:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1788878596.601418:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878596.601422:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878596.601423:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878596.601426:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878596.601427:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878596.601429:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:3.0:1788878596.601429:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878596.601432:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878596.601435:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878596.601435:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878596.601438:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878596.601439:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601442:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:3.0:1788878596.601445:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878596.601448:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601448:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878596.601450:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878596.601453:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304208425 02000000:00000001:3.0:1788878596.601454:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878596.601459:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601460:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878596.601461:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878596.601463:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601465:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878596.601465:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878596.601471:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878596.601473:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601474:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878596.601475:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304208425 02000000:00000001:3.0:1788878596.601476:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878596.601478:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601484:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601487:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00100000:3.0:1788878596.601488:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290079488/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878596.601494:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878596.601496:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878596.601499:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601502:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878596.601502:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601508:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878596.601517:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878596/real 1788878596] req@ffff94e50449e300 x1875765290079104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878596.601529:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878596.601531:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878596.601534:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878596.601536:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878596.601537:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601539:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878596.601541:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878596.601544:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290079104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878596.601555:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601557:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878596.601559:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601560:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878596.601561:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601563:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878596.601564:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878596.601566:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878596.601568:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878596.601570:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878596.601572:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290079104:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878596.601576:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601577:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878596.601579:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878596.601580:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878596.601583:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878596.601584:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878596.601586:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878596.601588:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878596.601590:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878596.601592:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878596.601593:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601595:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601602:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878596.601604:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601606:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878596.601608:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878596.601609:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878596.601611:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878596.601613:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878596.601615:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878596.601617:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878596.601619:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878596.601623:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290079232:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878596.601627:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878596.601629:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878596.601631:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878596.601633:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878596.601636:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601642:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878596.601643:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:0.0:1788878596.601647:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878596.601657:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878596.601660:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878596.601662:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878596.601663:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601679:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878596.601681:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601685:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304207883 00000100:00000001:0.0:1788878596.601687:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304208425 00000100:00000001:0.0:1788878596.601689:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304207883 00000100:00000001:0.0:1788878596.601691:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304208425 00000100:00000001:0.0:1788878596.601692:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878596.601693:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601694:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878596.601697:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878596/real 1788878596] req@ffff94e50449f100 x1875765290079232/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878596.601703:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878596.601704:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878596.601706:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878596.601707:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878596.601708:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601710:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878596.601711:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878596.601713:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290079232/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878596.601722:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601724:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878596.601725:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601726:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878596.601727:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601728:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878596.601739:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878596.601742:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878596.601745:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878596.601747:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878596.601751:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290079232:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878596.601754:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878596.601756:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878596.601757:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878596.601760:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878596.601762:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878596.601764:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878596.601767:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290079360:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878596.601770:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878596.601772:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878596.601774:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878596.601776:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878596.601779:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601786:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878596.601787:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:0.0:1788878596.601791:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878596.601800:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878596.601803:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878596.601805:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878596.601806:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601807:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878596.601809:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601811:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304207881 00000100:00000001:0.0:1788878596.601813:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304208425 00000100:00000001:0.0:1788878596.601814:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304207881 00000100:00000001:0.0:1788878596.601816:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304208425 00000100:00000001:0.0:1788878596.601817:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878596.601819:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601820:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878596.601822:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878596/real 1788878596] req@ffff94e50449ca80 x1875765290079360/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878596.601834:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878596.601835:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878596.601837:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878596.601838:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878596.601839:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601840:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878596.601841:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878596.601844:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290079360/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878596.601848:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601849:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878596.601851:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601852:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878596.601852:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601854:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878596.601864:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878596.601866:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878596.601868:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878596.601869:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878596.601872:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290079360:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878596.601874:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878596.601876:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878596.601876:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878596.601878:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878596.601879:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878596.601880:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878596.601882:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290079488:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878596.601884:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878596.601886:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878596.601887:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878596.601888:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878596.601890:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601892:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878596.601893:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:0.0:1788878596.601896:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878596.601926:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878596.601929:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878596.601930:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878596.601931:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601933:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878596.601934:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601937:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304207882 00000100:00000001:0.0:1788878596.601945:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304208425 00000100:00000001:0.0:1788878596.601946:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304207882 00000100:00000001:0.0:1788878596.601948:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304208425 00000100:00000001:0.0:1788878596.601949:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878596.601951:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.601952:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878596.601954:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878596/real 1788878596] req@ffff94e50449f800 x1875765290079488/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878596.601964:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878596.601965:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878596.601967:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878596.601968:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878596.601969:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601970:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878596.601971:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878596.601973:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290079488/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878596.601978:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601979:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878596.601980:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601981:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878596.601982:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.601983:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878596.601993:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878596.601994:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878596.601996:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878596.601998:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878596.602000:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290079488:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878596.602002:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.602004:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878596.602004:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878596.602005:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878596.602007:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878596.602008:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878596.602009:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878596.602010:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878596.602011:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878596.602013:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878596.602014:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.602015:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878596.602016:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878596.602017:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878596.602018:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878596.602019:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878596.602020:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878596.602021:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878596.602022:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878596.602022:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878596.602023:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.602024:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878596.602025:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878596.602025:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878596.602027:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878596.602027:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878596.602028:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878596.602029:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878596.602029:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878596.602030:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878596.602031:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.602032:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878596.602034:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878596.602035:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878596.602036:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878596.602037:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878597.368987:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878597.368994:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878597.368999:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878597.369000:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878597.369070:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878597.369078:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878597.369081:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878598.393038:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878598.393043:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878598.393047:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878598.393048:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878598.393076:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878598.393078:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878598.393081:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878599.417003:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878599.417010:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878599.417015:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878599.417017:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878599.417038:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878599.417041:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878599.417076:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878600.440958:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878600.440979:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878600.440984:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878600.440985:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878600.441003:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878600.441005:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878600.441008:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878601.465028:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878601.465035:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878601.465041:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878601.465044:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878601.465047:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878601.465052:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878601.465067:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878601.465070:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878601.465074:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878601.465078:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878601.465150:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878601.465155:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878601.465160:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878601.721035:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878601.721043:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878601.721045:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878601.721048:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878601.721051:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878601.721057:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9540 00000100:00080000:2.0:1788878601.721066:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878601.721068:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878601.721070:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878601.721076:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878601.721078:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878601.721082:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878601.721085:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878601.721087:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878601.721089:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878601.721094:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878601.721097:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878601.721100:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878601.721103:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878601.721106:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721109:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878601.721111:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878601.721135:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878601.721137:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721140:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878601.721143:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878601.721148:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721159:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878601.721167:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290079616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878601.721175:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878601.721179:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721193:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878601.721196:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878601.721199:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878601.721201:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878601.721203:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878601.721206:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878601.721210:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9540 00000100:00080000:2.0:1788878601.721215:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878601.721218:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878601.721220:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878601.721224:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788878601.721224:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878601.721229:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878601.721230:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878601.721233:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788878601.721233:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878601.721236:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1788878601.721239:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878601.721240:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721244:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788878601.721247:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788878601.721247:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:0.0:1788878601.721251:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788878601.721254:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:0.0:1788878601.721255:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721257:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878601.721261:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:0.0:1788878601.721262:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290079616:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878601.721266:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878601.721268:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878601.721272:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878601.721273:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788878601.721274:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:0.0:1788878601.721278:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788878601.721280:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788878601.721281:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:2.0:1788878601.721284:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721286:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878601.721289:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878601.721290:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878601.721294:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721295:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878601.721296:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878601.721298:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721303:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878601.721308:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290079744/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878601.721309:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878601.721314:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:2.0:1788878601.721318:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878601.721322:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878601.721322:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721327:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878601.721329:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878601.721332:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878601.721333:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878601.721334:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878601.721336:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878601.721338:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9540 00000100:00080000:2.0:1788878601.721342:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878601.721343:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878601.721344:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878601.721347:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878601.721348:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878601.721350:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878601.721352:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878601.721354:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878601.721355:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878601.721357:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878601.721358:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878601.721362:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:0.0:1788878601.721363:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:2.0:1788878601.721364:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878601.721368:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721369:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878601.721371:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878601.721373:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721374:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721375:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:2.0:1788878601.721378:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878601.721380:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788878601.721381:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721384:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721385:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878601.721388:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878601.721389:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:2.0:1788878601.721390:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721394:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721396:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721400:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304213545 00000100:00100000:2.0:1788878601.721401:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290079872/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878601.721409:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878601.721410:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721411:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878601.721413:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878601.721416:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878601.721418:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878601.721420:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304213545 00000100:00000001:2.0:1788878601.721420:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878601.721421:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878601.721427:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878601.721428:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1788878601.721430:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9540 00000100:00000001:0.0:1788878601.721433:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788878601.721434:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878601.721439:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721442:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878601.721447:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1788878601.721448:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878601/real 1788878601] req@ffff94e5045e5880 x1875765290079616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878601.721451:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878601.721457:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878601.721459:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878601.721462:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000400:00000001:0.0:1788878601.721465:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1788878601.721466:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000400:00000001:0.0:1788878601.721469:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878601.721470:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878601.721473:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878601.721474:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:2.0:1788878601.721477:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878601.721478:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878601.721482:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878601.721482:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721485:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1788878601.721486:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878601.721489:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878601.721491:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878601.721494:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878601.721495:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290079616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878601.721496:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878601.721499:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878601.721505:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721510:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721522:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878601.721524:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878601.721526:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788878601.721527:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721533:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878601.721534:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878601.721536:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878601.721537:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1788878601.721539:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721540:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.721544:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878601.721545:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721546:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878601.721551:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1788878601.721553:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290080000/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878601.721557:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878601.721562:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788878601.721568:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1788878601.721569:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290079616:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878601.721572:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878601.721578:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721579:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.721582:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878601.721584:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878601.721586:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878601.721588:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878601.721590:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878601.721591:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878601.721594:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878601.721595:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878601.721597:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878601.721598:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.721600:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.721606:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878601.721608:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721609:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878601.721610:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878601.721611:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878601.721612:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878601.721613:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878601.721615:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878601.721616:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878601.721618:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878601.721620:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290079744:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878601.721622:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878601.721624:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878601.721625:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878601.721626:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878601.721628:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721649:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878601.721650:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 00000100:00000001:0.0:1788878601.721654:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878601.721664:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878601.721667:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878601.721669:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878601.721682:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721683:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878601.721685:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721688:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304213003 00000100:00000001:0.0:1788878601.721690:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304213545 00000100:00000001:0.0:1788878601.721691:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304213003 00000100:00000001:0.0:1788878601.721693:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304213545 00000100:00000001:0.0:1788878601.721695:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878601.721696:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721697:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878601.721700:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878601/real 1788878601] req@ffff94e5045e5180 x1875765290079744/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878601.721705:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878601.721706:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878601.721708:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878601.721709:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878601.721710:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.721711:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878601.721712:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878601.721714:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290079744/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878601.721719:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.721721:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878601.721722:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.721723:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878601.721724:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.721725:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878601.721734:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878601.721736:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878601.721738:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878601.721739:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878601.721742:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290079744:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878601.721744:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878601.721745:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878601.721746:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878601.721748:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878601.721749:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878601.721750:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878601.721752:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290079872:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878601.721754:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878601.721755:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878601.721756:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878601.721758:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878601.721759:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721762:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878601.721763:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:0.0:1788878601.721765:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878601.721771:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878601.721774:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878601.721775:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878601.721776:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721777:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878601.721779:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721781:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304213001 00000100:00000001:0.0:1788878601.721783:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304213545 00000100:00000001:0.0:1788878601.721784:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304213001 00000100:00000001:0.0:1788878601.721786:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304213545 00000100:00000001:0.0:1788878601.721787:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878601.721788:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721789:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878601.721792:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878601/real 1788878601] req@ffff94e5045e4a80 x1875765290079872/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878601.721801:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878601.721802:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878601.721803:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878601.721804:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878601.721805:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.721806:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878601.721807:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878601.721809:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290079872/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878601.721813:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.721814:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878601.721815:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.721816:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878601.721817:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.721824:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878601.721832:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878601.721833:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878601.721835:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878601.721836:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878601.721838:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290079872:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878601.721851:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878601.721853:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878601.721858:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878601.721860:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878601.721861:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878601.721863:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878601.721867:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290080000:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878601.721870:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878601.721872:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878601.721874:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878601.721876:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878601.721879:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721893:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878601.721895:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:0.0:1788878601.721899:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878601.721933:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878601.721936:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878601.721938:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878601.721939:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721940:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878601.721942:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721945:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304213002 00000100:00000001:0.0:1788878601.721947:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304213545 00000100:00000001:0.0:1788878601.721948:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304213002 00000100:00000001:0.0:1788878601.721950:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304213545 00000100:00000001:0.0:1788878601.721952:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878601.721953:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.721954:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878601.721957:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878601/real 1788878601] req@ffff94e5045e7100 x1875765290080000/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878601.721961:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878601.721962:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878601.721963:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878601.721964:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878601.721965:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.721967:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878601.721968:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878601.721970:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290080000/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878601.721974:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.721976:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878601.721978:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.721979:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878601.721980:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.721981:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878601.721989:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878601.721991:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878601.721994:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878601.721995:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878601.721997:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290080000:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878601.721999:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.722000:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878601.722001:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878601.722002:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878601.722003:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878601.722004:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878601.722005:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878601.722007:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878601.722008:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878601.722009:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878601.722010:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.722011:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878601.722012:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878601.722013:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878601.722014:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878601.722014:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878601.722015:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878601.722016:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878601.722017:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878601.722018:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878601.722018:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.722019:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878601.722020:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878601.722021:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878601.722022:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878601.722022:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878601.722023:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878601.722025:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878601.722026:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878601.722027:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878601.722028:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.722029:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878601.722031:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878601.722033:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878601.722034:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878601.722035:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878602.488987:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878602.488993:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878602.488999:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878602.489000:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878602.489020:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878602.489023:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878602.489026:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878603.513020:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878603.513026:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878603.513028:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878603.513031:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878603.513035:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878603.513036:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878603.513038:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878603.513040:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878603.513042:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878603.513043:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878603.513097:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878603.513101:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878603.513104:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878604.536972:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878604.536986:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878604.536990:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878604.536991:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878604.537006:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878604.537008:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878604.537010:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878605.561039:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878605.561047:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878605.561054:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878605.561056:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878605.561197:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878605.561204:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878605.561208:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878606.585023:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878606.585028:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878606.585032:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878606.585033:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878606.585049:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878606.585051:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878606.585053:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878606.841360:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878606.841372:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878606.841381:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878606.841401:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878606.841406:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878606.841416:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878606.841441:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878606.841449:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878606.841468:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290080128:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878606.841482:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878606.841488:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878606.841495:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878606.841501:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878606.841511:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.841533:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878606.841541:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:0.0:1788878606.841584:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878606.841693:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878606.841709:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878606.841720:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878606.841726:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.841731:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878606.841739:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.841748:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304218665 00000100:00000001:0.0:1788878606.841768:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878606.841782:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304218665 00000100:00000001:0.0:1788878606.841796:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878606.841803:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878606.841812:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.841819:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878606.841835:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878606/real 1788878606] req@ffff94e51100aa00 x1875765290080128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878606.841864:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878606.841869:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878606.841877:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878606.841886:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878606.841910:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.841917:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878606.841921:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878606.841928:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290080128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878606.842022:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842031:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878606.842042:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842051:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878606.842057:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842063:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878606.842067:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878606.842078:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878606.842094:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878606.842105:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878606.842119:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290080128:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878606.842148:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842153:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878606.842155:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878606.842159:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878606.842172:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878606.842175:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878606.842178:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878606.842183:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878606.842203:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878606.842207:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878606.842209:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842212:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842219:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878606.842222:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.842225:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878606.842227:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878606.842229:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878606.842230:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878606.842232:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878606.842235:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878606.842238:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878606.842240:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878606.842244:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290080256:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878606.842248:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878606.842251:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878606.842253:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878606.842255:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878606.842286:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.842298:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878606.842301:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286516096 : -117763423035520 : ffff94e511009f80) 00000100:00000001:0.0:1788878606.842309:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878606.842335:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878606.842346:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878606.842351:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878606.842354:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.842358:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878606.842363:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.842372:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304218124 00000100:00000001:0.0:1788878606.842377:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304218665 00000100:00000001:0.0:1788878606.842382:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304218124 00000100:00000001:0.0:1788878606.842386:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304218665 00000100:00000001:0.0:1788878606.842389:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878606.842401:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.842405:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878606.842411:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878606/real 1788878606] req@ffff94e511009f80 x1875765290080256/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878606.842423:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878606.842425:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878606.842429:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878606.842449:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878606.842451:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842453:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878606.842455:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878606.842460:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009f80 x1875765290080256/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878606.842468:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842470:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878606.842473:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842475:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878606.842477:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842479:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878606.842491:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878606.842494:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878606.842516:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878606.842518:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878606.842523:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290080256:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878606.842527:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878606.842529:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878606.842531:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878606.842534:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878606.842536:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878606.842538:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878606.842542:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290080384:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878606.842550:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878606.842552:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878606.842554:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878606.842557:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878606.842561:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.842569:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878606.842570:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286512512 : -117763423039104 : ffff94e511009180) 00000100:00000001:0.0:1788878606.842575:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878606.842592:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878606.842635:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878606.842638:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878606.842640:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.842643:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878606.842647:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.842652:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304218122 00000100:00000001:0.0:1788878606.842655:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304218665 00000100:00000001:0.0:1788878606.842658:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304218122 00000100:00000001:0.0:1788878606.842661:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304218665 00000100:00000001:0.0:1788878606.842663:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878606.842666:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.842668:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878606.842674:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878606/real 1788878606] req@ffff94e511009180 x1875765290080384/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878606.842684:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878606.842686:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878606.842689:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878606.842691:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878606.842693:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842695:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878606.842697:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878606.842702:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009180 x1875765290080384/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878606.842719:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842722:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878606.842725:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842726:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878606.842728:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842730:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878606.842743:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878606.842746:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878606.842749:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878606.842752:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878606.842756:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290080384:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878606.842760:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878606.842762:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878606.842764:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878606.842766:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878606.842769:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878606.842771:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878606.842775:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290080512:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878606.842779:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878606.842782:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878606.842784:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878606.842786:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878606.842790:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.842796:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878606.842797:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508928 : -117763423042688 : ffff94e511008380) 00000100:00000001:0.0:1788878606.842801:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878606.842816:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878606.842821:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878606.842847:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878606.842849:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.842851:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878606.842855:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.842859:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304218122 00000100:00000001:0.0:1788878606.842861:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304218665 00000100:00000001:0.0:1788878606.842864:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304218122 00000100:00000001:0.0:1788878606.842866:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304218665 00000100:00000001:0.0:1788878606.842869:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878606.842871:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.842873:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878606.842877:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878606/real 1788878606] req@ffff94e511008380 x1875765290080512/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878606.842886:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878606.842887:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878606.842890:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878606.842892:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878606.842893:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842895:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878606.842897:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878606.842901:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008380 x1875765290080512/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878606.842928:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842930:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878606.842933:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842935:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878606.842937:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.842938:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878606.842986:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878606.842990:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878606.842995:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878606.842998:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878606.843002:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290080512:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878606.843007:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.843009:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878606.843011:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878606.843013:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878606.843016:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878606.843018:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878606.843020:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878606.843023:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878606.843025:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878606.843028:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878606.843030:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.843032:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878606.843033:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878606.843035:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878606.843037:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878606.843039:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878606.843040:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878606.843042:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878606.843043:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878606.843045:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878606.843047:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.843049:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878606.843051:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878606.843053:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878606.843057:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878606.843059:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878606.843061:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878606.843063:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878606.843066:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878606.843068:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878606.843071:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.843074:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878606.843078:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878606.843080:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878606.843082:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878606.843084:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878607.609011:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878607.609020:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878607.609026:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878607.609029:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878607.609056:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878607.609059:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878607.609062:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878608.632983:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878608.632991:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878608.632998:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878608.632999:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878608.633033:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878608.633038:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878608.633043:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878609.657042:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878609.657050:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878609.657057:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878609.657058:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878609.657211:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878609.657220:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878609.657227:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878610.681072:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878610.681081:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878610.681087:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878610.681088:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878610.681120:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878610.681124:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878610.681127:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878611.705058:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878611.705065:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878611.705071:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878611.705075:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878611.705078:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878611.705082:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878611.705085:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878611.705088:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878611.705090:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878611.705092:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878611.705120:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878611.705123:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878611.705126:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878611.961261:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878611.961269:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878611.961273:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878611.961279:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878611.961284:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878611.961291:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878611.961298:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878611.961308:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878611.961319:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290080640:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878611.961327:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878611.961335:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878611.961339:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878611.961347:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878611.961354:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.961373:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878611.961378:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286514304 : -117763423037312 : ffff94e511009880) 00000100:00000001:0.0:1788878611.961383:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878611.961430:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878611.961452:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878611.961461:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878611.961467:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.961474:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878611.961485:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.961498:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304223785 00000100:00000001:0.0:1788878611.961518:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878611.961528:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304223785 00000100:00000001:0.0:1788878611.961538:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878611.961545:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878611.961552:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.961576:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878611.961588:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878611/real 1788878611] req@ffff94e511009880 x1875765290080640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878611.961627:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878611.961631:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878611.961639:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878611.961645:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878611.961651:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.961655:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878611.961660:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878611.961668:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009880 x1875765290080640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878611.961679:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.961686:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878611.961695:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.961698:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878611.961703:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.961710:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878611.961713:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878611.961724:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878611.961729:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878611.961737:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878611.961748:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290080640:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878611.961756:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.961762:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878611.961766:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878611.961772:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878611.961778:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878611.961785:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878611.961789:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878611.961798:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878611.961801:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878611.961808:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878611.961811:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.961817:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.961829:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878611.961834:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.961840:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878611.961844:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878611.961849:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878611.961855:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878611.961862:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878611.961871:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878611.961878:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878611.961891:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878611.961959:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290080768:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878611.961975:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878611.961984:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878611.961990:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878611.961999:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878611.962010:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.962022:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878611.962029:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508928 : -117763423042688 : ffff94e511008380) 00000100:00000001:0.0:1788878611.962038:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878611.962064:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878611.962071:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878611.962075:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878611.962077:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.962080:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878611.962084:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.962089:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304223244 00000100:00000001:0.0:1788878611.962093:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304223785 00000100:00000001:0.0:1788878611.962095:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304223244 00000100:00000001:0.0:1788878611.962098:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304223785 00000100:00000001:0.0:1788878611.962100:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878611.962102:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.962104:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878611.962109:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878611/real 1788878611] req@ffff94e511008380 x1875765290080768/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878611.962119:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878611.962120:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878611.962123:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878611.962124:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878611.962126:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962128:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878611.962130:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878611.962134:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008380 x1875765290080768/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878611.962142:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962144:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878611.962146:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962148:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878611.962149:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962151:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878611.962163:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878611.962166:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878611.962169:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878611.962171:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878611.962175:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290080768:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878611.962178:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878611.962180:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878611.962182:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878611.962184:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878611.962186:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878611.962188:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878611.962191:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290080896:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878611.962195:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878611.962197:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878611.962198:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878611.962200:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878611.962203:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.962208:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878611.962209:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286512512 : -117763423039104 : ffff94e511009180) 00000100:00000001:0.0:1788878611.962213:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878611.962223:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878611.962227:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878611.962229:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878611.962231:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.962232:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878611.962235:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.962238:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304223242 00000100:00000001:0.0:1788878611.962240:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304223785 00000100:00000001:0.0:1788878611.962242:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304223242 00000100:00000001:0.0:1788878611.962245:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304223785 00000100:00000001:0.0:1788878611.962247:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878611.962249:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.962250:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878611.962254:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878611/real 1788878611] req@ffff94e511009180 x1875765290080896/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878611.962262:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878611.962263:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878611.962265:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878611.962267:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878611.962268:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962270:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878611.962271:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878611.962275:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009180 x1875765290080896/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878611.962281:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962283:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878611.962285:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962287:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878611.962288:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962290:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878611.962300:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878611.962302:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878611.962305:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878611.962307:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878611.962311:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290080896:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878611.962315:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962317:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878611.962318:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878611.962320:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878611.962322:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878611.962323:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878611.962324:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878611.962327:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878611.962328:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878611.962330:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878611.962331:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962333:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878611.962334:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878611.962336:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878611.962337:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878611.962339:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878611.962340:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878611.962341:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878611.962342:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878611.962344:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878611.962345:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962347:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962350:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878611.962352:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.962354:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878611.962355:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878611.962356:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878611.962357:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878611.962359:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878611.962361:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878611.962362:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878611.962364:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878611.962367:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290081024:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878611.962370:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878611.962372:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878611.962373:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878611.962375:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878611.962378:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.962382:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878611.962383:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286516096 : -117763423035520 : ffff94e511009f80) 00000100:00000001:0.0:1788878611.962387:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878611.962396:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878611.962400:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878611.962402:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878611.962403:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.962405:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878611.962408:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.962411:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304223242 00000100:00000001:0.0:1788878611.962413:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304223785 00000100:00000001:0.0:1788878611.962416:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304223242 00000100:00000001:0.0:1788878611.962418:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304223785 00000100:00000001:0.0:1788878611.962420:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878611.962423:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.962425:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878611.962431:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878611/real 1788878611] req@ffff94e511009f80 x1875765290081024/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878611.962445:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878611.962447:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878611.962451:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878611.962453:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878611.962456:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962459:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878611.962461:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878611.962467:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009f80 x1875765290081024/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878611.962478:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962482:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878611.962486:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962489:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878611.962492:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962495:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878611.962511:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878611.962516:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878611.962519:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878611.962522:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878611.962525:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290081024:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878611.962529:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962532:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878611.962533:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878611.962534:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878611.962537:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878611.962538:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878611.962540:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878611.962542:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878611.962543:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878611.962545:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878611.962547:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962549:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878611.962552:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878611.962554:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878611.962556:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878611.962557:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878612.729053:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878612.729060:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878612.729067:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878612.729068:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878612.729095:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878612.729098:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878612.729112:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878613.753016:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878613.753023:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878613.753025:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878613.753028:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878613.753033:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878613.753035:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878613.753037:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878613.753039:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878613.753042:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878613.753044:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878613.753133:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878613.753138:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878613.753143:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878614.776999:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878614.777005:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878614.777010:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878614.777011:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878614.777051:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878614.777054:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878614.777057:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878615.801009:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878615.801017:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878615.801023:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878615.801025:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878615.801051:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878615.801054:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878615.801058:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878616.825005:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878616.825013:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878616.825019:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878616.825021:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878616.825046:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878616.825050:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878616.825053:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878617.081304:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878617.081315:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878617.081322:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878617.081329:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878617.081335:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878617.081344:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878617.081351:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878617.081355:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878617.081366:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290081152:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878617.081379:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878617.081385:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878617.081391:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878617.081398:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878617.081409:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.081438:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878617.081444:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:0.0:1788878617.081469:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788878617.081558:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.081569:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304228905 00000100:00000001:0.0:1788878617.081589:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878617.081604:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304228905 00000100:00000001:0.0:1788878617.081613:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878617.081622:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878617.081628:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.081648:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.081656:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878617.081669:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878617.081674:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788878617.081682:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878617.081690:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878617.081697:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878617.081700:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878617.081706:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878617.081709:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878617.081713:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878617.081724:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878617.081736:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290081280:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878617.081752:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878617.081758:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878617.081763:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878617.081770:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878617.081780:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.081793:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878617.081799:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:0.0:1788878617.081806:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788878617.081825:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.081839:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304228363 00000100:00000001:0.0:1788878617.081848:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304228905 00000100:00000001:0.0:1788878617.081855:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304228363 00000100:00000001:0.0:1788878617.081863:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304228905 00000100:00000001:0.0:1788878617.081869:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878617.081872:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.081879:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.081885:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878617.081889:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878617.081894:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788878617.081935:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878617.081938:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878617.081940:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878617.081942:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878617.081943:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878617.081947:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878617.081950:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878617.081953:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878617.081963:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290081408:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878617.081973:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878617.081979:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878617.081984:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878617.081990:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878617.081997:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.082008:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878617.082014:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:0.0:1788878617.082021:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788878617.082039:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.082047:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304228362 00000100:00000001:0.0:1788878617.082052:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304228905 00000100:00000001:0.0:1788878617.082061:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304228362 00000100:00000001:0.0:1788878617.082072:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304228905 00000100:00000001:0.0:1788878617.082080:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878617.082086:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.082098:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.082106:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878617.082118:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878617.082119:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788878617.082125:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878617.082134:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878617.082139:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.082145:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.082290:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878617.082295:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878617.082298:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878617.082300:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878617.082303:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878617.082309:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878617.082313:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878617.082317:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878617.082325:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290081536:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878617.082333:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878617.082337:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878617.082341:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878617.082344:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878617.082352:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.082365:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878617.082368:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286517888 : -117763423033728 : ffff94e51100a680) 00000100:00000001:0.0:1788878617.082375:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788878617.082400:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.082407:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304228362 00000100:00000001:0.0:1788878617.082413:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304228905 00000100:00000001:0.0:1788878617.082418:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304228362 00000100:00000001:0.0:1788878617.082422:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304228905 00000100:00000001:0.0:1788878617.082426:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878617.082430:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.082436:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.082440:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878617.082448:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878617.082451:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788878617.082455:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878617.082458:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878617.082461:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878617.082465:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878617.848992:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878617.848999:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878617.849004:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878617.849006:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878617.849069:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878617.849073:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878617.849077:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878618.873064:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878618.873071:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878618.873078:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878618.873080:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878618.873150:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878618.873154:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878618.873158:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878619.897025:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878619.897032:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878619.897039:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878619.897041:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878619.897094:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878619.897098:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878619.897101:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788878620.153253:0:2351:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1788878620.153305:0:2351:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788878620.153321:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788878620.153335:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878620.153352:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878620.153355:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878620.153357:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:3.0:1788878620.153365:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788878620.153374:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878620.153376:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878620.153377:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878620.153379:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:3.0:1788878620.153383:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788878620.153386:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878620.153388:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878620.153389:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878620.153391:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:3.0:1788878620.153395:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788878620.153402:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878620.153406:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878620.153409:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878620.153411:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878620.153433:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878620.153440:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878620.153444:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878620.153458:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878617/real 1788878620] req@ffff94e51100a680 x1875765290081536/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878620.153477:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878620.153479:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878620.153487:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878620.153490:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878620.153491:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153494:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878620.153497:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878620.153521:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100a680 x1875765290081536/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878620.153539:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153544:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878620.153548:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153550:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878620.153552:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153556:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878620.153574:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878620.153580:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878620.153587:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878620.153591:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878620.153596:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290081536:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878620.153602:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878620.153608:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878617/real 1788878620] req@ffff94e51100b800 x1875765290081408/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878620.153620:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878620.153622:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878620.153626:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878620.153628:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878620.153630:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153633:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878620.153634:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878620.153640:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290081408/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878620.153650:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153652:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878620.153656:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153658:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878620.153660:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153663:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878620.153679:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878620.153683:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878620.153689:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878620.153692:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878620.153697:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290081408:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878620.153702:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878620.153708:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878617/real 1788878620] req@ffff94e51100b480 x1875765290081280/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878620.153719:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878620.153721:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878620.153724:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878620.153727:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878620.153728:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153731:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878620.153732:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878620.153738:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290081280/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878620.153748:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153750:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878620.153753:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153755:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878620.153757:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153759:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878620.153775:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878620.153779:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878620.153784:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878620.153786:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878620.153790:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290081280:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878620.153795:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878620.153801:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878617/real 1788878620] req@ffff94e51100aa00 x1875765290081152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878620.153811:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878620.153812:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878620.153816:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878620.153818:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878620.153820:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153822:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878620.153824:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878620.153828:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290081152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878620.153837:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153839:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878620.153842:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153844:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878620.153846:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153848:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878620.153851:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878620.153854:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878620.153858:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878620.153861:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878620.153865:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290081152:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878620.153871:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153874:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878620.153876:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878620.153880:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878620.153886:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878620.153889:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878620.153892:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878620.153897:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878620.153934:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878620.153939:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878620.153941:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153943:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878620.153946:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878620.153949:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878620.153952:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878620.153955:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878620.153957:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878620.153960:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878620.153962:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878620.153965:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878620.153967:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153969:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878620.153971:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878620.153973:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878620.153976:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878620.153977:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878620.153979:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878620.153982:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878620.153984:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878620.153986:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878620.153988:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.153991:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878620.153994:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878620.153997:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878620.154000:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878620.154002:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878620.154003:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878620.154006:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878620.154008:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878620.154009:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878620.154011:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.154014:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878620.154022:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878620.154025:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878620.154028:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878620.154030:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878620.921062:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878620.921071:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878620.921079:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878620.921098:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878620.921129:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878620.921132:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878620.921136:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878621.945001:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878621.945008:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878621.945015:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878621.945018:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878621.945022:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878621.945026:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878621.945029:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878621.945032:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878621.945035:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878621.945038:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878621.945130:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878621.945136:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878621.945141:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878622.201117:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878622.201127:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878622.201131:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878622.201136:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878622.201143:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878622.201154:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9561 00000100:00080000:3.0:1788878622.201167:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878622.201171:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878622.201174:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878622.201184:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878622.201188:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878622.201192:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878622.201196:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878622.201200:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878622.201202:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878622.201208:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878622.201212:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878622.201216:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878622.201221:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878622.201225:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.201228:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878622.201232:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878622.201244:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878622.201246:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.201250:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878622.201253:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878622.201258:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.201275:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878622.201289:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290081664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878622.201304:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878622.201310:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.201338:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878622.201343:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878622.201348:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878622.201350:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878622.201353:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878622.201359:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878622.201365:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9561 00000100:00080000:3.0:1788878622.201375:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878622.201380:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878622.201384:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878622.201390:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878622.201394:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878622.201398:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878622.201405:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878622.201414:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878622.201422:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878622.201433:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878622.201444:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878622.201452:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878622.201464:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878622.201475:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.201484:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878622.201518:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878622.201533:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.201542:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878622.201552:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878622.201560:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.201568:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878622.201574:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878622.201579:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.201598:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878622.201613:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290081792/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878622.201644:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878622.201650:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.201659:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878622.201665:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878622.201672:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878622.201676:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878622.201679:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878622.201684:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878622.201692:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9561 00000100:00080000:3.0:1788878622.201707:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878622.201726:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878622.201730:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878622.201739:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878622.201746:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878622.201752:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878622.201761:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878622.201773:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878622.201779:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878622.201789:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878622.201801:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878622.201811:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878622.201821:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878622.201827:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.201837:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878622.201843:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878622.201856:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.201861:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878622.201875:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878622.201881:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.201891:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878622.201899:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878622.201949:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.201961:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878622.201975:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290081920/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878622.202012:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878622.202022:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.202031:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878622.202042:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878622.202053:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878622.202063:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878622.202067:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878622.202079:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878622.202093:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9561 00000100:00080000:3.0:1788878622.202109:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878622.202119:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878622.202122:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878622.203520:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878622.203527:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878622.203532:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878622.203540:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878622.203546:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878622.203550:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878622.203561:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878622.203569:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878622.203575:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878622.203585:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878622.203591:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.203596:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878622.203601:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878622.203607:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.203610:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878622.203711:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878622.203715:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.203719:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878622.203723:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878622.203728:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.203739:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878622.203751:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290082048/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878622.203779:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878622.203785:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878622.203811:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878622.969013:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878622.969020:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878622.969026:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878622.969028:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878622.969054:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878622.969058:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878622.969061:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878623.993009:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878623.993019:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878623.993022:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878623.993026:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878623.993033:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878623.993035:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878623.993038:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878623.993041:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878623.993045:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878623.993047:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878623.993073:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878623.993076:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878623.993079:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878625.016993:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878625.016999:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878625.017004:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878625.017006:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878625.017026:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878625.017029:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878625.017031:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878626.041044:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878626.041052:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878626.041058:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878626.041060:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878626.041129:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878626.041135:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878626.041140:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878627.064987:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878627.064992:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878627.064996:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878627.064998:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878627.065018:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878627.065032:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878627.065035:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878627.321347:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878627.321356:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878627.321364:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878627.321370:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878627.321376:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878627.321384:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878627.321392:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878627.321401:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878627.321417:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290082176:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878627.321432:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878627.321440:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878627.321446:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878627.321479:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878627.321492:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.321528:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878627.321534:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286523264 : -117763423028352 : ffff94e51100bb80) 00000100:00000001:0.0:1788878627.321545:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878627.321604:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878627.321625:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878627.321635:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878627.321642:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.321646:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878627.321655:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.321668:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304239145 00000100:00000001:0.0:1788878627.321693:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878627.321712:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304239145 00000100:00000001:0.0:1788878627.321730:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878627.321742:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878627.321753:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.321764:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878627.321783:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878627/real 1788878627] req@ffff94e51100bb80 x1875765290082176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878627.321829:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878627.321836:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878627.321850:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878627.321861:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878627.321868:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.321874:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878627.321881:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878627.321899:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100bb80 x1875765290082176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878627.321965:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.321973:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878627.321981:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.321986:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878627.321991:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.322000:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878627.322005:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878627.322014:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878627.322028:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878627.322038:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878627.322050:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290082176:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878627.322065:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.322077:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878627.322082:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878627.322088:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878627.322096:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878627.322102:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878627.322107:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878627.322117:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878627.322124:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878627.322134:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878627.322144:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.322156:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.322189:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878627.322197:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.322209:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878627.322219:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878627.322224:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878627.322228:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878627.322233:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878627.322242:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878627.322249:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878627.322259:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878627.322274:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290082304:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878627.322286:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878627.322295:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878627.322299:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878627.322302:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878627.322309:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.322323:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878627.322326:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286509824 : -117763423041792 : ffff94e511008700) 00000100:00000001:0.0:1788878627.322332:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878627.322356:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878627.322362:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878627.322366:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878627.322368:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.322371:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878627.322374:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.322380:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304238604 00000100:00000001:0.0:1788878627.322383:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304239145 00000100:00000001:0.0:1788878627.322386:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304238604 00000100:00000001:0.0:1788878627.322390:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304239145 00000100:00000001:0.0:1788878627.322393:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878627.322396:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.322398:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878627.322404:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878627/real 1788878627] req@ffff94e511008700 x1875765290082304/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878627.322415:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878627.322417:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878627.322420:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878627.322422:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878627.322424:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.322427:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878627.322428:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878627.322433:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008700 x1875765290082304/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878627.322442:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.322445:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878627.322448:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.322479:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878627.322481:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.322484:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878627.322499:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878627.322502:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878627.322507:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878627.322509:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878627.322513:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290082304:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878627.322519:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878627.322521:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878627.322523:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878627.322525:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878627.322528:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878627.322531:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878627.322535:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290082432:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878627.322539:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878627.322542:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878627.322544:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878627.322546:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878627.322550:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.322558:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878627.322560:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286511616 : -117763423040000 : ffff94e511008e00) 00000100:00000001:0.0:1788878627.322565:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878627.322582:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878627.322587:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878627.322590:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878627.322592:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.322595:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878627.322598:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.322602:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304238602 00000100:00000001:0.0:1788878627.322605:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304239145 00000100:00000001:0.0:1788878627.322608:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304238602 00000100:00000001:0.0:1788878627.322611:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304239145 00000100:00000001:0.0:1788878627.322613:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878627.322615:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.322618:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878627.322622:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878627/real 1788878627] req@ffff94e511008e00 x1875765290082432/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878627.322632:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878627.322634:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878627.322636:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878627.322639:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878627.322640:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.322643:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878627.322644:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878627.322649:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008e00 x1875765290082432/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878627.322658:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.322661:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878627.322664:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.322666:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878627.322667:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.322669:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878627.322688:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878627.322693:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878627.322699:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878627.322703:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878627.322709:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290082432:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878627.322716:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.322720:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878627.322722:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878627.322726:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878627.322730:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878627.322733:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878627.322736:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878627.322741:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878627.322743:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878627.322747:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878627.322750:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.322754:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878627.322756:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878627.322759:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878627.322763:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878627.322765:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878627.322768:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878627.322772:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878627.322774:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878627.322777:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878627.322780:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.322801:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.322810:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878627.322812:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.322816:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878627.322819:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878627.322822:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878627.322825:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878627.322827:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878627.322832:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878627.322835:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878627.322839:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878627.322847:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290082560:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878627.322864:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878627.322868:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878627.322871:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878627.322875:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878627.322881:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.322892:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878627.322894:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508032 : -117763423043584 : ffff94e511008000) 00000100:00000001:0.0:1788878627.322927:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878627.322949:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878627.322955:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878627.322959:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878627.322961:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.322963:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878627.322967:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.322972:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304238603 00000100:00000001:0.0:1788878627.322975:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304239145 00000100:00000001:0.0:1788878627.322978:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304238603 00000100:00000001:0.0:1788878627.322980:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304239145 00000100:00000001:0.0:1788878627.322983:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878627.322999:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.323002:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878627.323007:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878627/real 1788878627] req@ffff94e511008000 x1875765290082560/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878627.323028:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878627.323029:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878627.323033:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878627.323035:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878627.323037:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.323039:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878627.323041:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878627.323046:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008000 x1875765290082560/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878627.323060:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.323063:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878627.323066:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.323068:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878627.323070:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.323072:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878627.323085:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878627.323087:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878627.323091:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878627.323094:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878627.323098:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290082560:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878627.323103:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.323105:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878627.323107:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878627.323109:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878627.323113:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878627.323116:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878627.323118:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878627.323122:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878627.323125:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878627.323129:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878627.323131:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.323136:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878627.323142:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878627.323145:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878627.323149:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878627.323152:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878628.089035:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878628.089043:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878628.089060:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878628.089062:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878628.089088:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878628.089091:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878628.089095:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878629.113020:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878629.113026:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878629.113031:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878629.113033:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878629.113056:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878629.113059:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878629.113062:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878630.137034:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878630.137042:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878630.137048:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878630.137050:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878630.137188:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878630.137191:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878630.137194:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878631.161027:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878631.161036:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878631.161042:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878631.161046:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878631.161049:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878631.161054:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878631.161057:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878631.161059:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878631.161062:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878631.161065:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878631.161129:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878631.161132:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878631.161136:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878632.185016:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878632.185024:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878632.185032:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878632.185034:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878632.185073:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878632.185076:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878632.185080:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878632.441122:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878632.441143:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878632.441149:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878632.441156:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878632.441166:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878632.441181:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9571 00000100:00080000:3.0:1788878632.441199:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878632.441204:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878632.441207:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878632.441220:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878632.441224:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878632.441242:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878632.441246:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878632.441249:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878632.441251:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878632.441259:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878632.441262:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878632.441267:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878632.441273:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878632.441277:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878632.441281:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878632.441284:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878632.441303:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878632.441306:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878632.441310:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878632.441314:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878632.441322:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878632.441341:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878632.441355:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290082688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878632.441376:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878632.441385:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878632.441437:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878632.441445:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878632.441452:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878632.441457:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878632.441462:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878632.441470:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878632.441479:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9571 00000100:00080000:3.0:1788878632.441488:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878632.441492:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878632.441496:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878632.441497:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878632.441510:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788878632.441515:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878632.441516:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878632.441524:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1788878632.441525:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878632.441529:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878632.441537:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:0.0:1788878632.441541:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1788878632.441553:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:0.0:1788878632.441554:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878632.441569:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878632.441573:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878632.441584:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:0.0:1788878632.441585:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290082688:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878632.441602:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788878632.441605:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:0.0:1788878632.441611:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878632.441613:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878632.441619:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878632.441620:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878632.441631:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878632.441640:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878632.441643:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878632.441659:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878632.441670:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878632.441678:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000008:00100000:3.0:1788878632.441680:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878632.441684:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000008:00000001:3.0:1788878632.441698:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.441699:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878632.441711:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878632.441721:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878632.441723:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878632.441727:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878632.441729:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878632.441733:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878632.441748:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788878632.441765:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1788878632.441766:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290082816/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878632.441790:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878632.441791:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878632.441802:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.441805:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878632.441812:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878632.441815:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878632.441825:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878632.441830:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878632.441832:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878632.441838:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:3.0:1788878632.441838:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878632.441853:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878632.441860:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9571 00000100:00000001:0.0:1788878632.441862:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878632.441870:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878632.441872:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304244265 00000100:00000001:3.0:1788878632.441879:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878632.441883:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878632.441889:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878632.441893:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1788878632.441894:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878632.441952:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304244266 00000100:00000001:0.0:1788878632.441960:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878632.441965:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878632.441970:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.441974:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1788878632.441985:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00100000:0.0:1788878632.441988:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878632/real 1788878632] req@ffff94e50449ce00 x1875765290082688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788878632.442004:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878632.442019:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878632.442022:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:0.0:1788878632.442028:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878632.442033:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1788878632.442041:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878632.442051:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.442055:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:3.0:1788878632.442056:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878632.442062:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878632.442066:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878632.442068:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878632.442072:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878632.442076:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878632.442082:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878632.442094:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:0.0:1788878632.442096:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290082688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788878632.442110:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878632.442114:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.442128:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:3.0:1788878632.442129:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.442138:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878632.442142:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.442144:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878632.442147:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.442161:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878632.442168:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878632.442171:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00080000:0.0:1788878632.442175:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1788878632.442176:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878632.442190:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878632.442196:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00080000:0.0:1788878632.442197:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878632.442204:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1788878632.442204:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878632.442223:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290082688:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878632.442226:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.442235:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.442245:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1788878632.442247:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290082944/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878632.442253:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878632.442262:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878632.442272:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1788878632.442273:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000020:00000001:0.0:1788878632.442282:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:3.0:1788878632.442283:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878632.442291:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788878632.442296:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878632.442300:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878632.442307:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:3.0:1788878632.442307:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878632.442323:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00080000:3.0:1788878632.442324:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878632.442335:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878632.442336:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878632.442347:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878632.442348:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878632.442367:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878632.442371:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1788878632.442378:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9571 00000100:00000001:0.0:1788878632.442379:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.442387:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878632.442395:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878632.442398:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1788878632.442401:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878632.442402:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878632.442407:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878632.442416:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878632.442442:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788878632.442448:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878632.442451:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:0.0:1788878632.442455:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878632.442476:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:0.0:1788878632.442477:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290082816:192.168.203.129@tcp:38:kworker.0 00000020:00000001:3.0:1788878632.442488:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878632.442491:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1788878632.442495:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:0.0:1788878632.442498:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878632.442508:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878632.442508:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:0.0:1788878632.442521:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878632.442528:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788878632.442530:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878632.442537:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878632.442544:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878632.442549:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00080000:3.0:1788878632.442561:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878632.442564:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878632.442576:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878632.442585:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878632.442592:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:0.0:1788878632.442606:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788878632.442609:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878632.442615:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878632.442618:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00100000:3.0:1788878632.442620:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878632.442626:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00000001:3.0:1788878632.442626:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.442634:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878632.442639:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.442640:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878632.442647:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.442655:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304243724 02000000:00000001:3.0:1788878632.442658:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878632.442661:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304244265 02000000:00000001:3.0:1788878632.442665:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.442667:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304243724 00000100:00000001:0.0:1788878632.442677:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304244266 00000100:00000001:3.0:1788878632.442681:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878632.442683:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878632.442690:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878632.442690:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878632.442701:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1788878632.442704:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878632.442710:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878632/real 1788878632] req@ffff94e50449d880 x1875765290082816/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878632.442723:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788878632.442726:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878632.442733:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878632.442738:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878632.442743:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00100000:3.0:1788878632.442745:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290083072/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878632.442748:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.442760:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878632.442765:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878632.442788:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1788878632.442789:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290082816/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878632.442800:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878632.442818:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.442821:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.442833:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878632.442839:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.442843:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878632.442847:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.442851:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878632.442879:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878632.442884:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878632.442890:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878632.442893:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878632.442898:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290082816:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878632.442936:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878632.442939:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878632.442942:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878632.442945:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878632.442948:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878632.442951:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878632.442955:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290082944:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878632.442960:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878632.442963:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878632.442966:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878632.442968:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878632.442973:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.442982:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878632.442984:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:0.0:1788878632.442990:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878632.443008:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878632.443014:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878632.443018:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878632.443020:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.443022:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878632.443025:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.443030:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304243723 00000100:00000001:0.0:1788878632.443033:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304244265 00000100:00000001:0.0:1788878632.443037:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304243723 00000100:00000001:0.0:1788878632.443040:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304244266 00000100:00000001:0.0:1788878632.443043:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878632.443045:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.443047:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878632.443052:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878632/real 1788878632] req@ffff94e50449c380 x1875765290082944/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878632.443063:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878632.443064:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878632.443067:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878632.443069:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878632.443071:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.443073:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878632.443075:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878632.443080:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290082944/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878632.443088:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.443091:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878632.443095:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.443097:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878632.443099:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.443101:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878632.443113:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878632.443116:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878632.443120:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878632.443123:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878632.443127:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290082944:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878632.443131:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.443133:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878632.443135:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878632.443137:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878632.443140:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878632.443141:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878632.443143:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878632.443146:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878632.443148:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878632.443151:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878632.443152:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.443154:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878632.443156:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878632.443157:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878632.443159:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878632.443161:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878632.443162:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878632.443164:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878632.443165:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878632.443167:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878632.443169:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.443171:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.443176:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878632.443177:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.443180:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878632.443182:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878632.443184:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878632.443186:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878632.443188:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878632.443191:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878632.443193:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878632.443196:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878632.443201:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290083072:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878632.443206:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878632.443209:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878632.443211:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878632.443214:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878632.443218:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.443224:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878632.443226:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:0.0:1788878632.443231:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878632.443249:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878632.443254:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878632.443257:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878632.443259:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.443262:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878632.443269:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.443276:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304243723 00000100:00000001:0.0:1788878632.443281:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304244265 00000100:00000001:0.0:1788878632.443286:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304243723 00000100:00000001:0.0:1788878632.443290:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304244266 00000100:00000001:0.0:1788878632.443294:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878632.443299:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.443304:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878632.443314:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878632/real 1788878632] req@ffff94e50449fb80 x1875765290083072/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878632.443333:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878632.443336:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878632.443342:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878632.443346:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878632.443348:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.443353:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878632.443356:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878632.443363:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290083072/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878632.443376:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.443379:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878632.443383:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.443385:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878632.443387:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.443389:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878632.443402:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878632.443405:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878632.443409:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878632.443411:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878632.443416:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290083072:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878632.443447:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.443449:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878632.443451:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878632.443453:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878632.443457:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878632.443458:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878632.443460:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878632.443463:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878632.443465:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878632.443468:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878632.443470:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.443472:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878632.443476:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878632.443478:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878632.443481:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878632.443483:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878633.209035:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878633.209044:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878633.209048:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878633.209053:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878633.209059:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878633.209062:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878633.209065:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878633.209068:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878633.209072:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878633.209074:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878633.209099:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878633.209102:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878633.209106:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878634.233028:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878634.233036:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878634.233044:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878634.233062:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878634.233122:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878634.233129:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878634.233135:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878635.257012:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878635.257023:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878635.257032:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878635.257035:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878635.257082:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878635.257086:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878635.257092:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878636.281051:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878636.281061:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878636.281070:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878636.281072:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878636.281102:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878636.281107:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878636.281112:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878637.304994:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878637.305000:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878637.305004:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878637.305005:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878637.305030:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878637.305033:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878637.305035:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878637.561472:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878637.561482:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878637.561488:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878637.561493:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878637.561495:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878637.561518:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878637.561537:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878637.561544:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878637.561562:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290083200:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878637.561580:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878637.561589:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878637.561593:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878637.561614:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878637.561636:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.561672:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878637.561678:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286519680 : -117763423031936 : ffff94e51100ad80) 00000100:00000001:0.0:1788878637.561690:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878637.561765:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878637.561795:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878637.561814:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878637.561823:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.561842:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878637.561859:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.561868:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304249385 00000100:00000001:0.0:1788878637.561889:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878637.561942:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304249385 00000100:00000001:0.0:1788878637.561952:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878637.561956:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878637.561959:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.561962:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878637.561971:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878637/real 1788878637] req@ffff94e51100ad80 x1875765290083200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878637.561984:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878637.561992:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878637.561999:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878637.562006:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878637.562016:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562035:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878637.562046:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878637.562065:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100ad80 x1875765290083200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878637.562094:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562104:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878637.562121:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562135:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878637.562150:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562188:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878637.562206:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878637.562238:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878637.562263:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878637.562288:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878637.562315:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290083200:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878637.562346:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562357:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878637.562361:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878637.562367:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878637.562376:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878637.562412:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878637.562418:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878637.562431:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878637.562436:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878637.562449:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878637.562451:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562456:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562467:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878637.562471:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.562476:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878637.562480:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878637.562484:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878637.562487:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878637.562491:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878637.562495:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878637.562499:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878637.562502:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878637.562509:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290083328:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878637.562530:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878637.562533:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878637.562535:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878637.562536:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878637.562539:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.562546:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878637.562548:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508032 : -117763423043584 : ffff94e511008000) 00000100:00000001:0.0:1788878637.562553:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878637.562569:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878637.562574:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878637.562576:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878637.562577:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.562579:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878637.562582:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.562585:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304248844 00000100:00000001:0.0:1788878637.562588:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304249385 00000100:00000001:0.0:1788878637.562590:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304248844 00000100:00000001:0.0:1788878637.562593:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304249385 00000100:00000001:0.0:1788878637.562594:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878637.562596:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.562598:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878637.562602:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878637/real 1788878637] req@ffff94e511008000 x1875765290083328/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878637.562610:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878637.562611:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878637.562613:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878637.562614:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878637.562616:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562617:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878637.562619:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878637.562622:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008000 x1875765290083328/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878637.562629:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562631:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878637.562633:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562634:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878637.562635:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562637:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878637.562648:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878637.562650:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878637.562653:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878637.562655:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878637.562657:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290083328:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878637.562661:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878637.562663:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878637.562664:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878637.562666:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878637.562668:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878637.562669:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878637.562672:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290083456:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878637.562674:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878637.562676:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878637.562678:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878637.562679:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878637.562681:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.562685:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878637.562686:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286511616 : -117763423040000 : ffff94e511008e00) 00000100:00000001:0.0:1788878637.562689:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878637.562698:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878637.562701:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878637.562703:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878637.562704:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.562706:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878637.562709:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.562725:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304248842 00000100:00000001:0.0:1788878637.562728:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304249385 00000100:00000001:0.0:1788878637.562731:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304248842 00000100:00000001:0.0:1788878637.562734:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304249385 00000100:00000001:0.0:1788878637.562737:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878637.562739:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.562741:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878637.562747:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878637/real 1788878637] req@ffff94e511008e00 x1875765290083456/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878637.562758:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878637.562760:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878637.562764:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878637.562766:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878637.562768:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562771:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878637.562773:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878637.562779:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008e00 x1875765290083456/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878637.562789:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562792:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878637.562794:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562796:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878637.562797:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562799:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878637.562827:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878637.562829:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878637.562833:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878637.562835:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878637.562839:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290083456:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878637.562844:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878637.562846:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878637.562847:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878637.562850:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878637.562852:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878637.562854:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878637.562857:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290083584:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878637.562861:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878637.562863:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878637.562865:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878637.562867:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878637.562870:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.562876:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878637.562878:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286509824 : -117763423041792 : ffff94e511008700) 00000100:00000001:0.0:1788878637.562882:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878637.562896:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878637.562900:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878637.562903:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878637.562905:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.562906:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878637.562909:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.562913:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304248842 00000100:00000001:0.0:1788878637.562915:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304249385 00000100:00000001:0.0:1788878637.562918:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304248842 00000100:00000001:0.0:1788878637.562920:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304249385 00000100:00000001:0.0:1788878637.562922:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878637.562924:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.562926:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878637.562930:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878637/real 1788878637] req@ffff94e511008700 x1875765290083584/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878637.562971:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878637.562973:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878637.562976:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878637.562978:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878637.562979:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562981:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878637.562983:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878637.562987:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008700 x1875765290083584/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878637.562994:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.562997:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878637.562999:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.563001:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878637.563002:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.563004:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878637.563014:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878637.563017:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878637.563020:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878637.563022:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878637.563025:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290083584:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878637.563029:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.563031:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878637.563033:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878637.563046:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878637.563048:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878637.563050:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878637.563051:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878637.563054:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878637.563055:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878637.563058:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878637.563059:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.563062:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878637.563063:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878637.563064:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878637.563067:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878637.563068:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878637.563069:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878637.563071:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878637.563072:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878637.563074:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878637.563075:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.563076:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878637.563078:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878637.563079:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878637.563080:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878637.563081:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878637.563082:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878637.563084:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878637.563085:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878637.563086:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878637.563087:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.563089:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878637.563093:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878637.563094:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878637.563096:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878637.563098:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878638.329040:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878638.329047:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878638.329064:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878638.329065:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878638.329117:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878638.329123:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878638.329128:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878639.352974:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878639.352984:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878639.352992:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878639.352993:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878639.353018:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878639.353020:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878639.353024:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878640.377020:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878640.377025:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878640.377040:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878640.377041:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878640.377061:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878640.377064:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878640.377066:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878641.401052:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878641.401059:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878641.401063:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878641.401066:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878641.401069:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878641.401072:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878641.401074:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878641.401076:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878641.401078:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878641.401080:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878641.401102:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878641.401105:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878641.401107:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878642.425092:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878642.425103:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878642.425110:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878642.425112:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878642.425224:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878642.425230:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878642.425234:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878642.681487:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878642.681503:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878642.681509:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878642.681514:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878642.681523:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878642.681554:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878642.681565:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878642.681572:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878642.681591:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290083712:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878642.681618:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878642.681630:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878642.681638:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878642.681649:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878642.681667:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.681708:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878642.681716:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286523264 : -117763423028352 : ffff94e51100bb80) 00000100:00000001:0.0:1788878642.681727:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878642.681823:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878642.681865:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878642.681874:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878642.681880:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.681897:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878642.681990:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.682009:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304254506 00000100:00000001:0.0:1788878642.682041:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878642.682066:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304254506 00000100:00000001:0.0:1788878642.682096:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878642.682118:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878642.682138:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.682147:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878642.682173:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878642/real 1788878642] req@ffff94e51100bb80 x1875765290083712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878642.682250:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878642.682269:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878642.682284:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878642.682303:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878642.682317:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.682331:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878642.682339:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878642.682384:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100bb80 x1875765290083712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878642.682436:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.682443:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878642.682448:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.682453:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878642.682456:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.682460:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878642.682463:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878642.682468:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878642.682475:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878642.682480:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878642.682486:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290083712:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878642.682493:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.682496:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878642.682499:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878642.682502:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878642.682520:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878642.682524:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878642.682526:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878642.682533:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878642.682535:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878642.682540:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878642.682542:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.682546:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.682556:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878642.682558:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.682561:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878642.682564:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878642.682566:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878642.682569:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878642.682571:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878642.682574:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878642.682577:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878642.682581:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878642.682584:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290083840:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878642.682589:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878642.682593:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878642.682595:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878642.682598:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878642.682602:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.682613:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878642.682614:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:0.0:1788878642.682621:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878642.682643:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878642.682650:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878642.682654:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878642.682656:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.682658:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878642.682662:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.682668:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304253964 00000100:00000001:0.0:1788878642.682672:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304254506 00000100:00000001:0.0:1788878642.682676:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304253964 00000100:00000001:0.0:1788878642.682679:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304254506 00000100:00000001:0.0:1788878642.682681:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878642.682684:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.682686:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878642.682692:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878642/real 1788878642] req@ffff94e51100aa00 x1875765290083840/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878642.682702:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878642.682710:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878642.682713:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878642.682715:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878642.682717:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.682720:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878642.682722:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878642.682726:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290083840/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878642.682736:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.682739:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878642.682742:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.682744:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878642.682745:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.682747:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878642.682770:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878642.682773:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878642.682777:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878642.682779:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878642.682784:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290083840:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878642.682789:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878642.682792:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878642.682794:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878642.682797:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878642.682799:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878642.682801:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878642.682805:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290083968:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878642.682809:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878642.682811:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878642.682813:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878642.682816:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878642.682820:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.682827:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878642.682829:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:0.0:1788878642.682833:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878642.682845:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878642.682850:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878642.682852:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878642.682854:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.682857:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878642.682870:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.682874:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304253962 00000100:00000001:0.0:1788878642.682878:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304254506 00000100:00000001:0.0:1788878642.682881:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304253962 00000100:00000001:0.0:1788878642.682884:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304254506 00000100:00000001:0.0:1788878642.682886:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878642.682889:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.682891:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878642.682895:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878642/real 1788878642] req@ffff94e51100b480 x1875765290083968/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878642.682978:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878642.682981:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878642.682987:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878642.682991:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878642.682994:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.682998:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878642.683001:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878642.683011:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290083968/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878642.683026:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.683030:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878642.683033:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.683035:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878642.683037:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.683040:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878642.683058:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878642.683061:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878642.683065:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878642.683068:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878642.683073:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290083968:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878642.683078:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878642.683080:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878642.683082:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878642.683085:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878642.683088:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878642.683090:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878642.683094:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290084096:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878642.683098:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878642.683101:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878642.683104:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878642.683106:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878642.683110:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.683118:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878642.683120:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:0.0:1788878642.683125:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878642.683142:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878642.683148:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878642.683151:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878642.683153:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.683155:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878642.683159:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.683164:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304253963 00000100:00000001:0.0:1788878642.683168:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304254506 00000100:00000001:0.0:1788878642.683173:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304253963 00000100:00000001:0.0:1788878642.683178:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304254506 00000100:00000001:0.0:1788878642.683183:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878642.683187:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.683191:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878642.683199:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878642/real 1788878642] req@ffff94e51100b800 x1875765290084096/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878642.683214:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878642.683217:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878642.683223:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878642.683226:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878642.683229:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.683233:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878642.683236:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878642.683255:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290084096/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878642.683270:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.683275:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878642.683279:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.683281:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878642.683283:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.683285:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878642.683308:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878642.683313:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878642.683320:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878642.683324:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878642.683331:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290084096:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878642.683339:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.683343:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878642.683346:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878642.683370:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878642.683376:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878642.683379:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878642.683381:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878642.683386:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878642.683389:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878642.683393:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878642.683396:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.683401:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878642.683403:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878642.683405:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878642.683410:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878642.683412:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878642.683413:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878642.683416:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878642.683418:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878642.683420:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878642.683422:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.683424:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878642.683425:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878642.683426:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878642.683428:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878642.683430:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878642.683431:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878642.683433:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878642.683435:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878642.683437:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878642.683439:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.683441:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878642.683446:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878642.683448:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878642.683451:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878642.683453:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878643.449050:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878643.449062:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878643.449067:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878643.449073:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878643.449082:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878643.449086:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878643.449090:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878643.449095:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878643.449102:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878643.449106:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878643.449135:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878643.449138:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878643.449142:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878644.473048:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878644.473055:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878644.473078:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878644.473080:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878644.473108:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878644.473112:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878644.473116:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878645.497069:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878645.497080:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878645.497091:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878645.497094:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878645.497131:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878645.497136:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878645.497142:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878646.521024:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878646.521032:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878646.521037:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878646.521039:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878646.521149:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878646.521167:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878646.521172:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878647.544996:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878647.545003:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878647.545008:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878647.545009:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878647.545028:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878647.545031:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878647.545034:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878647.801307:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878647.801335:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878647.801342:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878647.801348:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878647.801360:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878647.801367:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878647.801375:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878647.801385:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878647.801397:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290084224:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878647.801413:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878647.801423:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878647.801430:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878647.801440:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878647.801452:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.801479:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878647.801487:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286517888 : -117763423033728 : ffff94e51100a680) 00000100:00000001:0.0:1788878647.801500:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878647.801566:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878647.801603:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878647.801614:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878647.801621:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.801630:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878647.801650:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.801663:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304259625 00000100:00000001:0.0:1788878647.801683:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878647.801699:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304259625 00000100:00000001:0.0:1788878647.801725:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878647.801733:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878647.801739:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.801743:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878647.801760:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878647/real 1788878647] req@ffff94e51100a680 x1875765290084224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878647.801814:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878647.801818:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878647.801827:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878647.801833:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878647.801838:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.801859:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878647.801865:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878647.801891:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100a680 x1875765290084224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878647.802017:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802028:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878647.802048:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802053:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878647.802057:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802065:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878647.802069:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878647.802076:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878647.802088:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878647.802095:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878647.802105:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290084224:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878647.802122:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802128:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878647.802143:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878647.802146:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878647.802152:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878647.802157:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878647.802162:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878647.802171:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878647.802177:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878647.802183:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878647.802188:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802195:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802217:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878647.802222:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.802227:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878647.802230:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878647.802233:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878647.802236:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878647.802238:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878647.802241:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878647.802245:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878647.802248:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878647.802254:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290084352:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878647.802259:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878647.802262:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878647.802264:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878647.802266:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878647.802271:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.802281:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878647.802283:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286516096 : -117763423035520 : ffff94e511009f80) 00000100:00000001:0.0:1788878647.802288:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878647.802309:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878647.802343:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878647.802347:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878647.802349:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.802351:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878647.802355:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.802372:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304259084 00000100:00000001:0.0:1788878647.802376:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304259625 00000100:00000001:0.0:1788878647.802379:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304259084 00000100:00000001:0.0:1788878647.802382:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304259625 00000100:00000001:0.0:1788878647.802385:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878647.802388:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.802391:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878647.802397:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878647/real 1788878647] req@ffff94e511009f80 x1875765290084352/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878647.802409:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878647.802411:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878647.802414:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878647.802417:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878647.802419:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802421:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878647.802423:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878647.802428:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009f80 x1875765290084352/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878647.802436:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802439:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878647.802442:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802444:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878647.802445:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802448:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878647.802465:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878647.802468:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878647.802473:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878647.802475:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878647.802481:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290084352:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878647.802489:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878647.802491:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878647.802494:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878647.802498:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878647.802501:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878647.802505:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878647.802511:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290084480:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878647.802522:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878647.802526:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878647.802529:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878647.802533:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878647.802539:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.802550:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878647.802553:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286512512 : -117763423039104 : ffff94e511009180) 00000100:00000001:0.0:1788878647.802560:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878647.802585:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878647.802603:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878647.802607:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878647.802609:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.802611:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878647.802615:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.802619:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304259082 00000100:00000001:0.0:1788878647.802623:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304259625 00000100:00000001:0.0:1788878647.802626:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304259082 00000100:00000001:0.0:1788878647.802629:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304259625 00000100:00000001:0.0:1788878647.802631:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878647.802634:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.802636:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878647.802641:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878647/real 1788878647] req@ffff94e511009180 x1875765290084480/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878647.802651:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878647.802652:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878647.802655:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878647.802667:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878647.802669:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802672:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878647.802673:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878647.802678:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009180 x1875765290084480/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878647.802695:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802698:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878647.802701:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802703:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878647.802704:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802706:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878647.802721:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878647.802724:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878647.802733:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878647.802736:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878647.802740:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290084480:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878647.802744:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878647.802746:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878647.802748:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878647.802751:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878647.802753:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878647.802756:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878647.802759:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290084608:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878647.802763:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878647.802766:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878647.802768:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878647.802770:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878647.802773:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.802779:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878647.802781:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508928 : -117763423042688 : ffff94e511008380) 00000100:00000001:0.0:1788878647.802785:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878647.802798:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878647.802803:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878647.802805:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878647.802807:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.802810:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878647.802812:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.802816:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304259082 00000100:00000001:0.0:1788878647.802819:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304259625 00000100:00000001:0.0:1788878647.802822:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304259082 00000100:00000001:0.0:1788878647.802824:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304259625 00000100:00000001:0.0:1788878647.802827:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878647.802829:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.802831:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878647.802836:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878647/real 1788878647] req@ffff94e511008380 x1875765290084608/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878647.802845:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878647.802846:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878647.802848:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878647.802850:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878647.802852:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802854:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878647.802855:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878647.802859:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008380 x1875765290084608/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878647.802868:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802871:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878647.802873:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802875:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878647.802877:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802879:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878647.802897:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878647.802902:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878647.802946:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878647.802950:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878647.802959:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290084608:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878647.802966:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.802970:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878647.802973:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878647.802976:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878647.802981:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878647.802984:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878647.802987:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878647.802993:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878647.802994:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878647.802997:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878647.802999:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.803001:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878647.803003:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878647.803005:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878647.803007:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878647.803009:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878647.803010:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878647.803012:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878647.803014:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878647.803016:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878647.803017:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.803019:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878647.803021:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878647.803022:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878647.803024:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878647.803025:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878647.803026:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878647.803028:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878647.803029:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878647.803031:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878647.803033:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.803035:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878647.803039:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878647.803041:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878647.803044:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878647.803045:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878648.569035:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878648.569040:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878648.569044:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878648.569046:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878648.569065:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878648.569067:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878648.569069:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878649.593061:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878649.593072:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878649.593082:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878649.593085:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878649.593155:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878649.593161:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878649.593166:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878650.617040:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878650.617047:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878650.617053:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878650.617054:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878650.617227:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878650.617233:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878650.617239:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878651.641054:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878651.641063:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878651.641071:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878651.641076:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878651.641079:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878651.641084:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878651.641088:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878651.641091:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878651.641094:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878651.641097:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878651.641177:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878651.641181:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878651.641185:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878652.665065:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878652.665076:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878652.665083:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878652.665085:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878652.665116:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878652.665120:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878652.665124:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878652.921446:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878652.921459:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878652.921470:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878652.921481:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878652.921488:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878652.921505:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878652.921516:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878652.921527:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878652.921542:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290084736:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878652.921560:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878652.921571:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878652.921578:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878652.921585:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878652.921596:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.921628:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878652.921635:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286514304 : -117763423037312 : ffff94e511009880) 00000100:00000001:0.0:1788878652.921646:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878652.921712:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878652.921735:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878652.921744:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878652.921751:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.921755:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878652.921765:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.921772:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304264745 00000100:00000001:0.0:1788878652.921791:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878652.921809:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304264745 00000100:00000001:0.0:1788878652.921821:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878652.921828:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878652.921834:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.921843:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878652.921856:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878652/real 1788878652] req@ffff94e511009880 x1875765290084736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878652.921899:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878652.921975:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878652.921986:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878652.921996:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878652.922003:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922012:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878652.922017:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878652.922030:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009880 x1875765290084736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878652.922081:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922089:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878652.922102:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922107:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878652.922112:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922117:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878652.922121:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878652.922125:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878652.922132:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878652.922136:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878652.922142:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290084736:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878652.922151:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922155:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878652.922157:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878652.922161:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878652.922166:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878652.922168:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878652.922172:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878652.922178:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878652.922180:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878652.922184:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878652.922186:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922189:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922198:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878652.922201:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.922204:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878652.922206:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878652.922208:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878652.922210:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878652.922212:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878652.922215:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878652.922219:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878652.922222:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878652.922226:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290084864:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878652.922230:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878652.922232:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878652.922234:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878652.922237:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878652.922241:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.922249:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878652.922251:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508928 : -117763423042688 : ffff94e511008380) 00000100:00000001:0.0:1788878652.922257:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878652.922274:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878652.922312:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878652.922316:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878652.922318:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.922321:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878652.922325:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.922332:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304264204 00000100:00000001:0.0:1788878652.922337:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304264745 00000100:00000001:0.0:1788878652.922340:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304264204 00000100:00000001:0.0:1788878652.922343:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304264745 00000100:00000001:0.0:1788878652.922346:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878652.922348:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.922350:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878652.922356:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878652/real 1788878652] req@ffff94e511008380 x1875765290084864/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878652.922368:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878652.922369:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878652.922373:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878652.922375:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878652.922377:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922379:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878652.922381:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878652.922386:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008380 x1875765290084864/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878652.922397:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922401:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878652.922406:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922409:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878652.922412:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922415:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878652.922442:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878652.922448:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878652.922456:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878652.922461:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878652.922468:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290084864:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878652.922477:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878652.922480:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878652.922482:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878652.922487:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878652.922490:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878652.922495:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878652.922501:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290084992:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878652.922506:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878652.922508:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878652.922511:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878652.922513:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878652.922517:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.922526:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878652.922528:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286512512 : -117763423039104 : ffff94e511009180) 00000100:00000001:0.0:1788878652.922533:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878652.922551:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878652.922557:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878652.922561:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878652.922563:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.922565:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878652.922569:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.922573:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304264202 00000100:00000001:0.0:1788878652.922576:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304264745 00000100:00000001:0.0:1788878652.922579:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304264202 00000100:00000001:0.0:1788878652.922582:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304264745 00000100:00000001:0.0:1788878652.922585:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878652.922587:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.922590:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878652.922595:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878652/real 1788878652] req@ffff94e511009180 x1875765290084992/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878652.922604:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878652.922606:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878652.922609:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878652.922611:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878652.922613:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922615:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878652.922617:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878652.922621:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009180 x1875765290084992/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878652.922630:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922633:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878652.922636:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922638:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878652.922640:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922642:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878652.922660:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878652.922663:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878652.922666:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878652.922668:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878652.922673:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290084992:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878652.922678:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878652.922680:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878652.922682:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878652.922684:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878652.922686:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878652.922688:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878652.922692:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290085120:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878652.922696:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878652.922698:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878652.922700:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878652.922702:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878652.922705:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.922710:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878652.922712:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286516096 : -117763423035520 : ffff94e511009f80) 00000100:00000001:0.0:1788878652.922716:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878652.922727:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878652.922731:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878652.922734:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878652.922736:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.922738:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878652.922741:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.922745:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304264202 00000100:00000001:0.0:1788878652.922748:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304264745 00000100:00000001:0.0:1788878652.922751:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304264202 00000100:00000001:0.0:1788878652.922754:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304264745 00000100:00000001:0.0:1788878652.922756:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878652.922759:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.922761:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878652.922765:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878652/real 1788878652] req@ffff94e511009f80 x1875765290085120/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878652.922783:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878652.922784:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878652.922787:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878652.922789:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878652.922790:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922792:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878652.922794:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878652.922798:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009f80 x1875765290085120/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878652.922806:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922808:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878652.922811:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922813:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878652.922815:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922817:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878652.922835:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878652.922838:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878652.922841:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878652.922844:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878652.922848:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290085120:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878652.922852:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922855:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878652.922856:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878652.922858:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878652.922861:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878652.922862:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878652.922864:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878652.922867:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878652.922868:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878652.922871:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878652.922872:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922875:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878652.922876:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878652.922878:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878652.922880:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878652.922882:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878652.922883:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878652.922886:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878652.922887:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878652.922889:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878652.922890:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922892:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878652.922893:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878652.922895:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878652.922897:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878652.922898:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878652.922899:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878652.922901:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878652.922902:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878652.922933:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878652.922935:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922938:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878652.922942:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878652.922944:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878652.922946:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878652.922948:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878653.688994:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878653.689006:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878653.689009:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878653.689013:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878653.689019:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878653.689022:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878653.689025:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878653.689027:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878653.689032:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878653.689034:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878653.689058:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878653.689062:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878653.689066:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878654.713015:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878654.713022:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878654.713027:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878654.713029:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878654.713090:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878654.713093:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878654.713096:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878655.737000:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878655.737006:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878655.737011:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878655.737012:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878655.737042:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878655.737045:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878655.737047:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878656.761004:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878656.761009:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878656.761012:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878656.761013:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878656.761040:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878656.761042:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878656.761044:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878657.785029:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878657.785035:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878657.785040:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878657.785041:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878657.785088:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878657.785091:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878657.785093:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878658.041042:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878658.041052:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878658.041055:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878658.041059:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878658.041064:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878658.041072:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9596 00000100:00080000:3.0:1788878658.041084:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878658.041086:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878658.041088:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878658.041097:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878658.041100:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878658.041103:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878658.041105:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878658.041107:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878658.041108:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878658.041112:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878658.041114:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878658.041116:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878658.041119:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878658.041122:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878658.041123:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878658.041125:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878658.041138:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878658.041140:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878658.041143:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878658.041145:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878658.041151:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878658.041165:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878658.041174:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290085248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878658.041183:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878658.041188:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878658.041200:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878658.041203:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878658.041205:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878658.041206:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878658.041207:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878658.041210:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878658.041212:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9596 00000100:00080000:3.0:1788878658.041215:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878658.041217:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878658.041219:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878658.041221:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878658.041223:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878658.041225:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878658.041226:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878658.041227:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878658.041228:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878658.041231:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1788878658.041231:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878658.041233:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878658.041237:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878658.041238:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878658.041253:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878658.041256:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788878658.041257:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878658.041258:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878658.041264:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878658.041266:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878658.041269:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:0.0:1788878658.041270:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878658.041275:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788878658.041277:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878658.041281:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290085248:192.168.203.129@tcp:250:kworker.0 00000008:00000001:3.0:1788878658.041283:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878658.041289:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041290:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878658.041296:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878658.041297:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878658.041299:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878658.041301:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878658.041302:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878658.041307:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878658.041310:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878658.041310:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878658.041315:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878658.041322:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878658.041328:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290085376/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878658.041331:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878658.041335:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:3.0:1788878658.041340:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878658.041342:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878658.041344:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878658.041348:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878658.041350:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878658.041353:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878658.041354:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878658.041355:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878658.041358:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878658.041360:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9596 00000100:00080000:3.0:1788878658.041363:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878658.041364:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878658.041365:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878658.041368:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878658.041369:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878658.041371:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878658.041373:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878658.041375:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878658.041376:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878658.041379:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878658.041381:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878658.041384:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:0.0:1788878658.041385:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788878658.041389:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878658.041394:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878658.041396:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878658.041399:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878658.041400:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00000001:3.0:1788878658.041403:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041406:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878658.041408:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041410:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041413:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878658.041418:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878658.041418:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878658.041422:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041425:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304269865 00000100:00000001:3.0:1788878658.041428:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878658.041432:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878658.041434:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041437:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878658.041441:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041445:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304269865 00000100:00100000:3.0:1788878658.041449:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290085504/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878658.041453:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878658.041459:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878658.041460:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878658.041462:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878658.041464:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041466:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788878658.041468:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878658.041471:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878658.041475:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878658/real 1788878658] req@ffff94e50449df80 x1875765290085248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878658.041475:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878658.041478:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878658.041481:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878658.041487:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878658.041490:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9596 00000100:00080000:3.0:1788878658.041493:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878658.041496:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878658.041499:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:0.0:1788878658.041503:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1788878658.041503:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878658.041505:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000001:0.0:1788878658.041507:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1788878658.041509:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878658.041512:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878658.041513:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878658.041514:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878658.041517:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878658.041517:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878658.041520:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788878658.041521:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878658.041525:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878658.041525:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041528:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878658.041529:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878658.041533:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788878658.041537:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290085248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788878658.041538:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878658.041543:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878658.041546:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878658.041551:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041553:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878658.041554:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041562:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878658.041562:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878658.041565:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041568:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878658.041568:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878658.041571:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878658.041572:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788878658.041573:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041575:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.041578:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878658.041579:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041580:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878658.041586:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788878658.041586:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290085632/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878658.041592:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878658.041597:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788878658.041599:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878658.041603:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878658.041604:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290085248:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878658.041606:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041612:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.041614:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878658.041616:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878658.041619:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878658.041622:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878658.041624:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878658.041626:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878658.041629:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878658.041631:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878658.041633:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878658.041634:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.041636:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.041643:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878658.041645:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041646:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878658.041648:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878658.041649:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878658.041650:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878658.041651:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878658.041653:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878658.041654:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878658.041656:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878658.041658:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290085376:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878658.041660:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878658.041661:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878658.041662:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878658.041664:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878658.041668:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041672:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878658.041673:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000100:00000001:0.0:1788878658.041677:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878658.041687:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878658.041692:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878658.041693:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878658.041694:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041695:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878658.041697:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041701:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304269323 00000100:00000001:0.0:1788878658.041703:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304269865 00000100:00000001:0.0:1788878658.041704:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304269323 00000100:00000001:0.0:1788878658.041706:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304269865 00000100:00000001:0.0:1788878658.041708:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878658.041709:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041711:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878658.041714:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878658/real 1788878658] req@ffff94e50449e680 x1875765290085376/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878658.041720:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878658.041721:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878658.041722:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878658.041724:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878658.041725:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.041727:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878658.041728:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878658.041732:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290085376/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878658.041737:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.041738:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878658.041740:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.041741:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878658.041742:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.041743:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878658.041757:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878658.041759:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878658.041762:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878658.041763:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878658.041766:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290085376:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878658.041769:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878658.041770:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878658.041771:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878658.041773:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878658.041774:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878658.041775:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878658.041778:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290085504:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878658.041780:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878658.041781:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878658.041783:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878658.041784:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878658.041786:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041790:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878658.041791:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:0.0:1788878658.041794:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878658.041803:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878658.041808:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878658.041810:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878658.041812:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041813:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878658.041815:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041818:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304269321 00000100:00000001:0.0:1788878658.041820:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304269865 00000100:00000001:0.0:1788878658.041822:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304269321 00000100:00000001:0.0:1788878658.041824:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304269865 00000100:00000001:0.0:1788878658.041826:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878658.041828:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041829:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878658.041834:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878658/real 1788878658] req@ffff94e50449fb80 x1875765290085504/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878658.041843:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878658.041845:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878658.041848:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878658.041850:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878658.041851:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.041853:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878658.041855:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878658.041860:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290085504/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878658.041869:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.041872:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878658.041875:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.041877:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878658.041879:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.041881:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878658.041897:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878658.041900:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878658.041925:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878658.041927:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878658.041930:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290085504:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878658.041932:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878658.041934:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878658.041935:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878658.041936:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878658.041938:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878658.041939:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878658.041941:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290085632:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878658.041943:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878658.041945:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878658.041946:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878658.041948:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878658.041950:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041954:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878658.041955:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:0.0:1788878658.041958:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878658.041968:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878658.041972:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878658.041974:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878658.041975:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041977:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878658.041978:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041981:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304269322 00000100:00000001:0.0:1788878658.041983:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304269865 00000100:00000001:0.0:1788878658.041985:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304269322 00000100:00000001:0.0:1788878658.041987:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304269865 00000100:00000001:0.0:1788878658.041989:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878658.041990:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.041991:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878658.041994:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878658/real 1788878658] req@ffff94e50449c380 x1875765290085632/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878658.042002:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878658.042003:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878658.042005:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878658.042008:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878658.042009:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.042010:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878658.042011:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878658.042015:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290085632/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878658.042021:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.042023:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878658.042025:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.042026:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878658.042027:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.042028:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878658.042047:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878658.042049:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878658.042052:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878658.042054:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878658.042056:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290085632:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878658.042060:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.042061:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878658.042063:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878658.042065:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878658.042067:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878658.042068:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878658.042069:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878658.042070:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878658.042071:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878658.042072:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878658.042073:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.042075:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878658.042076:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878658.042077:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878658.042080:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878658.042081:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878658.042081:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878658.042083:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878658.042084:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878658.042086:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878658.042087:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.042088:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878658.042089:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878658.042090:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878658.042091:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878658.042092:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878658.042092:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878658.042094:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878658.042096:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878658.042097:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878658.042098:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.042099:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878658.042102:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878658.042103:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878658.042105:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878658.042107:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878658.808986:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878658.808992:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878658.808997:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878658.808998:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878658.809060:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878658.809064:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878658.809068:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878659.833100:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878659.833107:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878659.833112:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878659.833113:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878659.833139:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878659.833141:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878659.833144:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878660.857056:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878660.857066:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878660.857073:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878660.857075:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878660.857111:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878660.857116:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878660.857122:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878661.880996:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878661.881001:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878661.881004:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878661.881007:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878661.881009:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878661.881012:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878661.881014:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878661.881015:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878661.881017:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878661.881018:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878661.881035:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878661.881037:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878661.881039:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878662.904965:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878662.904970:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878662.904973:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878662.904974:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878662.905016:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878662.905019:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878662.905023:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878663.161059:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878663.161073:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878663.161077:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878663.161084:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878663.161094:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878663.161110:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9602 00000100:00080000:2.0:1788878663.161131:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878663.161138:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878663.161142:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878663.161159:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878663.161164:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878663.161171:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878663.161177:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878663.161181:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878663.161184:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878663.161194:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878663.161200:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878663.161229:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878663.161236:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878663.161243:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878663.161248:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878663.161253:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878663.161275:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878663.161278:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878663.161282:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878663.161286:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878663.161295:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878663.161317:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878663.161336:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290085760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878663.161359:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878663.161368:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878663.161399:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878663.161405:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878663.161410:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878663.161412:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878663.161415:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878663.161419:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878663.161424:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9602 00000100:00080000:2.0:1788878663.161431:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878663.161435:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878663.161437:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878663.161447:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878663.161448:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788878663.161450:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878663.161456:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878663.161458:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788878663.161461:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878663.161464:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878663.161466:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878663.161471:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878663.161473:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878663.161482:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878663.161486:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:0.0:1788878663.161492:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788878663.161492:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878663.161501:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878663.161502:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878663.161511:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878663.161516:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788878663.161522:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290085760:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878663.161524:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878663.161534:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878663.161536:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:2.0:1788878663.161544:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788878663.161545:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878663.161551:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788878663.161555:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878663.161557:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788878663.161566:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878663.161574:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878663.161575:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878663.161583:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878663.161587:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878663.161589:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878663.161592:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.161595:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878663.161599:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000100:00000001:2.0:1788878663.161601:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.161604:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1788878663.161623:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290085888/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878663.161648:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878663.161663:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878663.161665:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878663.161669:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878663.161673:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878663.161675:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.161677:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878663.161689:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878663.161694:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.161696:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304274985 00000100:00080000:2.0:1788878663.161707:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878663.161712:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1788878663.161715:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878663.161720:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878663.161721:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304274985 00000100:00080000:2.0:1788878663.161724:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878663.161727:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878663.161731:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878663.161733:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878663.161740:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878663.161741:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9602 00000100:00000001:0.0:1788878663.161746:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1788878663.161755:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1788878663.161760:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878663/real 1788878663] req@ffff94e5045e7800 x1875765290085760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878663.161762:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878663.161767:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878663.161781:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878663.161786:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878663.161793:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000400:00000001:0.0:1788878663.161795:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878663.161800:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1788878663.161806:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878663.161808:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878663.161818:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878663.161820:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878663.161828:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878663.161829:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.161845:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1788878663.161847:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878663.161853:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.161854:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878663.161856:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878663.161863:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878663.161870:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878663.161872:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290085760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878663.161873:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878663.161880:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878663.161892:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878663.161942:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878663.161950:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878663.161952:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878663.161955:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878663.161962:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878663.161972:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.161975:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.161979:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878663.161984:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.161985:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.161997:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878663.162004:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878663.162005:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290086016/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878663.162010:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878663.162015:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878663.162030:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788878663.162043:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00080000:0.0:1788878663.162044:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878663.162052:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162054:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788878663.162058:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878663.162065:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290085760:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1788878663.162065:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878663.162083:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878663.162085:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878663.162092:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878663.162092:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878663.162106:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878663.162108:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878663.162112:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00080000:2.0:1788878663.162119:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9602 02000000:00000001:0.0:1788878663.162120:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878663.162132:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:2.0:1788878663.162133:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000020:00000001:0.0:1788878663.162141:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:2.0:1788878663.162141:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000020:00000001:0.0:1788878663.162147:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:2.0:1788878663.162147:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:0.0:1788878663.162159:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:2.0:1788878663.162159:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878663.162170:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1788878663.162172:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:2.0:1788878663.162184:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878663.162186:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878663.162199:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788878663.162199:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878663.162224:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878663.162232:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878663.162236:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878663.162237:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878663.162244:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878663.162251:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878663.162252:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878663.162261:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878663.162263:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162265:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878663.162271:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788878663.162273:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878663.162277:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878663.162281:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878663.162282:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878663.162288:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878663.162289:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878663.162295:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878663.162299:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:0.0:1788878663.162303:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290085888:192.168.203.129@tcp:38:kworker.0 00000008:00100000:2.0:1788878663.162307:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878663.162308:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878663.162312:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878663.162315:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878663.162323:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:2.0:1788878663.162323:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878663.162329:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878663.162333:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162338:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878663.162341:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 00000100:00000001:0.0:1788878663.162346:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1788878663.162347:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878663.162353:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878663.162358:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878663.162362:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000800:00000100:0.0:1788878663.162368:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:2.0:1788878663.162371:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162374:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878663.162377:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878663.162381:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162385:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878663.162387:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162391:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304274444 00000100:00000001:2.0:1788878663.162393:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162394:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304274985 00000100:00000001:0.0:1788878663.162410:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304274444 00000100:00100000:2.0:1788878663.162418:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290086144/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878663.162420:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304274985 00000100:00000001:0.0:1788878663.162433:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878663.162447:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878663.162448:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878663.162452:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162457:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878663.162462:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878663.162472:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878663/real 1788878663] req@ffff94e5045e6300 x1875765290085888/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878663.162486:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878663.162487:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878663.162495:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878663.162497:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878663.162498:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162499:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878663.162501:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878663.162504:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290085888/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878663.162508:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162510:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878663.162512:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162513:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878663.162513:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162514:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878663.162527:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878663.162529:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878663.162531:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878663.162532:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878663.162534:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290085888:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878663.162537:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878663.162539:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878663.162540:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878663.162542:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878663.162544:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878663.162545:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878663.162547:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290086016:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878663.162550:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878663.162552:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878663.162553:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878663.162555:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878663.162558:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162562:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878663.162563:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:0.0:1788878663.162566:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878663.162575:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878663.162578:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878663.162580:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878663.162581:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162582:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878663.162584:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162587:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304274442 00000100:00000001:0.0:1788878663.162588:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304274985 00000100:00000001:0.0:1788878663.162590:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304274442 00000100:00000001:0.0:1788878663.162592:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304274985 00000100:00000001:0.0:1788878663.162594:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878663.162595:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162596:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878663.162598:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878663/real 1788878663] req@ffff94e5045e5500 x1875765290086016/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878663.162603:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878663.162604:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878663.162605:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878663.162606:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878663.162607:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162608:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878663.162609:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878663.162612:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290086016/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878663.162616:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162617:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878663.162619:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162620:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878663.162621:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162622:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878663.162631:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878663.162633:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878663.162635:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878663.162637:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878663.162638:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290086016:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878663.162641:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162642:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878663.162643:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878663.162644:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878663.162645:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878663.162646:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878663.162647:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878663.162649:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878663.162650:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878663.162651:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878663.162652:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162654:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878663.162654:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878663.162655:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878663.162656:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878663.162657:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878663.162657:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878663.162658:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878663.162659:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878663.162660:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878663.162661:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162662:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162664:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878663.162665:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162667:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878663.162668:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878663.162668:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878663.162669:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878663.162671:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878663.162672:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878663.162673:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878663.162675:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878663.162677:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290086144:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878663.162679:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878663.162680:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878663.162681:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878663.162682:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878663.162683:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162686:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878663.162687:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:0.0:1788878663.162690:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878663.162696:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878663.162698:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878663.162699:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878663.162700:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162701:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878663.162703:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162705:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304274442 00000100:00000001:0.0:1788878663.162707:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304274985 00000100:00000001:0.0:1788878663.162709:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304274442 00000100:00000001:0.0:1788878663.162710:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304274985 00000100:00000001:0.0:1788878663.162712:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878663.162713:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162714:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878663.162716:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878663/real 1788878663] req@ffff94e5045e5c00 x1875765290086144/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878663.162721:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878663.162722:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878663.162723:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878663.162725:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878663.162726:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162727:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878663.162728:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878663.162730:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290086144/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878663.162735:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162736:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878663.162737:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162738:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878663.162739:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162740:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878663.162749:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878663.162751:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878663.162753:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878663.162754:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878663.162756:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290086144:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878663.162758:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162759:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878663.162760:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878663.162761:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878663.162762:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878663.162763:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878663.162763:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878663.162765:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878663.162766:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878663.162767:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878663.162768:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162769:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878663.162771:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878663.162772:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878663.162773:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878663.162774:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878663.929013:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878663.929026:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878663.929031:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878663.929038:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878663.929047:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878663.929051:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878663.929055:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878663.929060:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878663.929067:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878663.929070:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878663.929099:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878663.929102:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878663.929106:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878664.953045:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878664.953053:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878664.953059:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878664.953061:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878664.953100:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878664.953104:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878664.953108:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878665.977010:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878665.977017:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878665.977023:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878665.977025:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878665.977049:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878665.977064:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878665.977067:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878667.001056:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878667.001064:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878667.001071:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878667.001073:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878667.001175:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878667.001207:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878667.001213:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878668.024968:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878668.024974:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878668.024977:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878668.024978:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878668.024993:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878668.024996:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878668.024998:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878668.281044:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878668.281055:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878668.281058:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878668.281062:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878668.281067:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878668.281094:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9607 00000100:00080000:0.0:1788878668.281111:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878668.281117:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878668.281121:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878668.281133:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878668.281137:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878668.281144:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878668.281149:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878668.281153:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878668.281156:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878668.281165:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878668.281192:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878668.281212:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878668.281215:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878668.281217:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.281219:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878668.281221:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878668.281230:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878668.281231:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.281234:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878668.281236:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878668.281239:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.281248:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878668.281256:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290086272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878668.281286:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878668.281292:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.281316:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878668.281321:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878668.281328:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878668.281332:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878668.281335:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878668.281340:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878668.281348:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9607 00000100:00080000:0.0:1788878668.281354:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878668.281358:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878668.281363:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878668.281371:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878668.281375:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878668.281381:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788878668.281393:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878668.281416:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878668.281420:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878668.281428:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878668.281430:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878668.281433:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878668.281447:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878668.281461:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.281486:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878668.281496:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878668.281501:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.281505:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878668.281516:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878668.281518:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.281524:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878668.281541:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878668.281551:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.281566:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878668.281576:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290086400/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878668.281596:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878668.281601:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.281609:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878668.281619:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878668.281625:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878668.281629:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878668.281632:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878668.281637:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878668.281644:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9607 00000100:00080000:0.0:1788878668.281658:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878668.281667:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878668.281670:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878668.281684:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878668.281692:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878668.281696:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878668.281701:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878668.281705:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878668.281709:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878668.281716:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878668.281737:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878668.281743:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878668.281770:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878668.281780:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.281789:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878668.281794:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878668.281806:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.281811:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878668.281840:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878668.281844:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.281850:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878668.281853:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878668.281856:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.281862:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878668.281871:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290086528/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878668.281904:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878668.281908:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.281915:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878668.281920:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878668.281925:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878668.281928:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878668.281931:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878668.281982:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878668.282000:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9607 00000100:00080000:0.0:1788878668.282041:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878668.282045:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878668.282047:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878668.282050:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878668.282053:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878668.282056:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788878668.282063:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878668.282066:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878668.282069:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878668.282075:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878668.282083:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878668.282113:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878668.282120:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878668.282125:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.282130:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878668.282135:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878668.282150:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.282194:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878668.282205:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878668.282209:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.282214:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878668.282219:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878668.282229:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.282239:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878668.282251:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290086656/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878668.282281:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878668.282290:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878668.282302:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878669.049080:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878669.049087:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878669.049092:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878669.049094:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878669.049118:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878669.049121:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878669.049124:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878670.073011:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878670.073018:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878670.073025:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878670.073026:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878670.073052:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878670.073055:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878670.073059:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878671.097039:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878671.097049:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878671.097057:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878671.097063:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878671.097068:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878671.097076:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878671.097082:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878671.097087:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878671.097091:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878671.097097:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878671.097274:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878671.097281:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878671.097286:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878672.121031:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878672.121038:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878672.121044:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878672.121046:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878672.121071:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878672.121074:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878672.121077:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878673.144982:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878673.144989:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878673.144992:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878673.144995:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878673.144999:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878673.145002:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878673.145004:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878673.145006:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878673.145010:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878673.145012:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878673.145032:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878673.145035:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878673.145038:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878673.401065:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878673.401076:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878673.401079:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878673.401083:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878673.401088:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878673.401095:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9612 00000100:00080000:3.0:1788878673.401107:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878673.401109:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878673.401111:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878673.401120:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878673.401123:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878673.401127:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878673.401129:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878673.401132:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878673.401133:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878673.401185:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878673.401188:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878673.401191:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878673.401194:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878673.401197:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401199:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878673.401202:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878673.401213:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878673.401215:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401217:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878673.401220:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878673.401224:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401235:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878673.401244:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290086784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878673.401254:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878673.401258:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401273:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878673.401276:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878673.401280:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878673.401281:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878673.401283:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878673.401286:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878673.401288:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9612 00000100:00080000:3.0:1788878673.401291:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878673.401293:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878673.401296:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878673.401300:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878673.401302:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878673.401306:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878673.401310:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878673.401313:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878673.401315:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878673.401323:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878673.401332:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878673.401340:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878673.401351:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878673.401361:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401367:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878673.401376:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878673.401387:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401393:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878673.401399:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878673.401403:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401405:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878673.401407:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878673.401410:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401417:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878673.401430:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290086912/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878673.401460:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878673.401463:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401468:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878673.401472:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878673.401476:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878673.401478:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878673.401480:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878673.401484:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878673.401493:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9612 00000100:00080000:3.0:1788878673.401504:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878673.401509:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878673.401512:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878673.401518:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878673.401524:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878673.401532:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878673.401538:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878673.401544:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878673.401547:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878673.401555:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878673.401564:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878673.401567:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878673.401571:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878673.401574:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401581:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878673.401584:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878673.401587:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401589:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878673.401595:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878673.401597:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401600:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878673.401606:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878673.401609:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401619:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878673.401652:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290087040/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878673.401667:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878673.401673:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401679:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878673.401687:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878673.401691:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878673.401697:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878673.401701:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878673.401705:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878673.401714:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9612 00000100:00080000:3.0:1788878673.401722:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878673.401727:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878673.401732:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878673.401738:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878673.401742:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878673.401747:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878673.401754:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878673.401759:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878673.401761:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878673.401782:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878673.401784:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878673.401791:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878673.401795:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878673.401801:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401807:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878673.401811:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878673.401816:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401820:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878673.401830:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878673.401834:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401839:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878673.401843:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878673.401847:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401855:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878673.401868:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290087168/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878673.401883:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878673.401888:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878673.401894:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878674.169074:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878674.169082:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878674.169088:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878674.169090:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878674.169116:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878674.169119:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878674.169122:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878675.193069:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878675.193077:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878675.193083:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878675.193085:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878675.193208:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878675.193214:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878675.193219:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878676.216994:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878676.217000:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878676.217031:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878676.217034:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878676.217060:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878676.217062:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878676.217066:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878677.241037:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878677.241043:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878677.241047:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878677.241049:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878677.241078:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878677.241081:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878677.241084:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878678.264971:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878678.264976:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878678.264980:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878678.264981:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878678.264997:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878678.264999:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878678.265001:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878678.521052:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878678.521061:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878678.521064:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878678.521067:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878678.521071:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878678.521078:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9617 00000100:00080000:2.0:1788878678.521088:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878678.521090:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878678.521093:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878678.521118:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878678.521122:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878678.521127:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878678.521130:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878678.521133:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878678.521135:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878678.521141:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878678.521145:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878678.521149:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878678.521165:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878678.521169:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521172:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878678.521176:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878678.521189:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878678.521190:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521193:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878678.521196:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878678.521201:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521214:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878678.521226:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4380 x1875765290087296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878678.521241:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878678.521246:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521265:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878678.521269:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878678.521272:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878678.521273:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878678.521275:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878678.521277:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878678.521280:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9617 00000100:00080000:2.0:1788878678.521284:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878678.521287:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878678.521288:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878678.521291:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878678.521293:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878678.521295:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878678.521297:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878678.521299:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878678.521301:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878678.521304:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878678.521305:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878678.521308:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878678.521316:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878678.521320:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521324:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878678.521331:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878678.521336:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521341:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878678.521349:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878678.521353:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521356:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878678.521360:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878678.521364:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521372:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878678.521382:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5f80 x1875765290087424/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878678.521395:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878678.521397:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521401:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878678.521403:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878678.521406:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878678.521408:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878678.521412:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878678.521420:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878678.521425:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9617 00000100:00080000:2.0:1788878678.521429:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878678.521430:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878678.521432:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878678.521434:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878678.521435:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878678.521437:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878678.521439:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878678.521441:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878678.521442:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878678.521445:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878678.521448:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878678.521450:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878678.521452:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878678.521454:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521455:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878678.521457:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878678.521462:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521466:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878678.521473:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878678.521475:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521477:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878678.521481:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878678.521483:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521489:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878678.521501:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290087552/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878678.521515:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878678.521519:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521521:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878678.521523:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878678.521528:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878678.521531:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878678.521534:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878678.521541:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878678.521546:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9617 00000100:00080000:2.0:1788878678.521552:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878678.521555:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878678.521556:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878678.521558:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878678.521560:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878678.521561:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878678.521564:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878678.521572:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878678.521576:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878678.521584:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878678.521591:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878678.521597:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878678.521602:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878678.521607:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521609:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878678.521614:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878678.521618:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521620:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878678.521628:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878678.521631:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521635:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878678.521639:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878678.521643:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521649:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878678.521661:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290087680/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878678.521676:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878678.521683:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878678.521689:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878679.289033:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878679.289038:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878679.289041:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878679.289042:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878679.289134:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878679.289156:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878679.289160:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878680.313014:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878680.313020:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878680.313024:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878680.313025:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878680.313043:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878680.313045:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878680.313048:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878681.337065:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878681.337072:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878681.337077:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878681.337080:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878681.337094:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878681.337097:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878681.337100:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878681.337114:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878681.337115:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878681.337117:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878681.337141:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878681.337144:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878681.337147:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878682.361100:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878682.361111:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878682.361119:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878682.361122:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878682.361156:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878682.361161:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878682.361165:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878683.385036:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878683.385045:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878683.385047:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878683.385051:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878683.385056:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878683.385058:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878683.385060:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878683.385061:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878683.385064:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878683.385065:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878683.385151:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878683.385154:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878683.385158:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878683.641116:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878683.641134:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878683.641138:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878683.641143:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878683.641149:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878683.641159:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9622 00000100:00080000:3.0:1788878683.641173:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878683.641176:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878683.641178:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878683.641187:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878683.641190:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878683.641193:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878683.641195:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878683.641197:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878683.641198:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878683.641203:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878683.641206:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878683.641209:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878683.641214:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878683.641218:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641222:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878683.641225:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878683.641242:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878683.641245:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641250:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878683.641253:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878683.641260:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641274:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878683.641286:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290087808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878683.641296:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878683.641300:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641316:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878683.641319:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878683.641322:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878683.641324:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878683.641325:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878683.641328:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878683.641341:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9622 00000100:00080000:3.0:1788878683.641344:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878683.641346:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878683.641347:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878683.641350:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878683.641351:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878683.641353:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878683.641356:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878683.641358:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878683.641359:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878683.641363:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878683.641365:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878683.641366:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878683.641372:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878683.641373:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641376:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878683.641384:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878683.641389:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641394:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878683.641400:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878683.641403:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641405:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878683.641409:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878683.641412:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641418:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878683.641427:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290087936/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878683.641437:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878683.641441:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641447:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878683.641450:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878683.641459:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878683.641461:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878683.641463:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878683.641467:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878683.641470:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9622 00000100:00080000:3.0:1788878683.641476:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878683.641482:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878683.641487:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878683.641495:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878683.641497:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878683.641500:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878683.641503:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878683.641507:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878683.641508:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878683.641513:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878683.641517:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878683.641519:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878683.641523:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878683.641526:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641528:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878683.641531:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878683.641533:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641534:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878683.641539:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878683.641540:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641542:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878683.641544:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878683.641548:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641555:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878683.641561:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290088064/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878683.641572:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878683.641575:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641578:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878683.641583:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878683.641590:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878683.641592:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878683.641596:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878683.641602:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878683.641607:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9622 00000100:00080000:3.0:1788878683.641613:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878683.641617:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878683.641619:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878683.641622:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878683.641625:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878683.641629:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878683.641633:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878683.641639:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878683.641642:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878683.641646:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878683.641650:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878683.641654:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878683.641658:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878683.641662:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641666:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878683.641669:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878683.641673:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641677:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878683.641683:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878683.641687:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641691:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878683.641704:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878683.641708:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641715:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878683.641725:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290088192/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878683.641734:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878683.641738:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878683.641742:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878684.409058:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878684.409092:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878684.409114:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878684.409116:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878684.409150:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878684.409154:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878684.409158:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878685.433007:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878685.433013:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878685.433017:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878685.433019:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878685.433037:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878685.433040:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878685.433042:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878686.457013:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878686.457020:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878686.457025:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878686.457027:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878686.457064:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878686.457067:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878686.457071:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788878686.713221:0:2352:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1788878686.713268:0:2352:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788878686.713282:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788878686.713294:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878686.713310:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878686.713312:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878686.713325:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788878686.713333:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788878686.713337:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878686.713338:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878686.713339:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878686.713341:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788878686.713350:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788878686.713356:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878686.713360:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878686.713363:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878686.713366:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878686.713373:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000100:0.0:1788878686.713374:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788878686.713379:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878686.713384:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788878686.713385:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878686.713391:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878686.713393:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878686.713397:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878683/real 1788878686] req@ffff94e50449c700 x1875765290087936/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878686.713398:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000400:00000001:2.0:1788878686.713416:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878686.713419:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878686.713428:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878686.713434:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878686.713438:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713443:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878686.713445:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878686.713451:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290087936/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878686.713460:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713463:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878686.713466:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713468:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878686.713470:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713473:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878686.713490:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878686.713495:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878686.713502:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878686.713506:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878686.713512:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290087936:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878686.713519:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878686.713525:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878683/real 1788878686] req@ffff94e50449f800 x1875765290088064/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878686.713538:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878686.713541:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878686.713545:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878686.713548:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878686.713550:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713554:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878686.713556:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878686.713561:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290088064/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878686.713570:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713574:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878686.713578:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713580:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878686.713581:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713583:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878686.713596:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878686.713607:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878686.713613:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878686.713616:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878686.713620:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290088064:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878686.713625:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878686.713630:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878683/real 1788878686] req@ffff94e50449c000 x1875765290087808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878686.713640:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878686.713642:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878686.713647:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878686.713649:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878686.713650:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713652:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878686.713654:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878686.713658:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290087808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878686.713665:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713668:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878686.713670:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713672:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878686.713673:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713675:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878686.713677:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878686.713680:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878686.713684:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878686.713686:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878686.713689:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290087808:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878686.713693:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878686.713696:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878686.713698:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878686.713702:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878686.713708:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878686.713711:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878686.713714:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878686.713718:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878686.713721:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878686.713725:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878686.713726:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713728:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878686.713729:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878686.713730:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878686.713732:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878686.713733:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878686.713734:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878686.713737:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878686.713738:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878686.713740:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878686.713741:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713742:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878686.713743:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878686.713745:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878686.713746:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878686.713748:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878686.713749:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878686.713750:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878686.713751:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878686.713753:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878686.713754:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713756:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878686.713758:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878686.713760:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878686.713761:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878686.713766:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878683/real 1788878686] req@ffff94e50449ca80 x1875765290088192/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878686.713776:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878686.713777:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878686.713781:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878686.713783:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878686.713784:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713786:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878686.713788:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878686.713792:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290088192/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878686.713799:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713801:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878686.713803:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713805:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878686.713807:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713809:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878686.713824:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878686.713827:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878686.713831:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878686.713834:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878686.713837:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290088192:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878686.713842:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713844:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878686.713845:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878686.713846:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878686.713849:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878686.713851:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878686.713852:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878686.713854:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878686.713856:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878686.713858:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878686.713860:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713862:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878686.713870:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878686.713872:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878686.713874:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878686.713876:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878687.480992:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878687.480998:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878687.481003:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878687.481004:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878687.481090:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878687.481095:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878687.481099:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878688.505016:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878688.505026:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878688.505056:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878688.505058:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878688.505094:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878688.505097:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878688.505101:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878688.761094:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878688.761109:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878688.761114:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878688.761119:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878688.761134:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878688.761142:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9627 00000100:00080000:3.0:1788878688.761154:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878688.761159:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878688.761161:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878688.761170:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878688.761174:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878688.761178:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878688.761181:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878688.761183:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878688.761185:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878688.761191:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878688.761195:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878688.761199:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878688.761204:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878688.761210:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878688.761214:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878688.761218:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878688.761233:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878688.761236:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878688.761240:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878688.761244:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878688.761250:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878688.761266:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878688.761281:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290088320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878688.761299:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878688.761306:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878688.761328:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878688.761333:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878688.761339:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878688.761341:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878688.761343:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878688.761347:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878688.761351:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9627 00000100:00080000:3.0:1788878688.761360:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878688.761364:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878688.761366:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878688.761371:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878688.761374:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788878688.761380:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788878688.761380:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878688.761393:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878688.761397:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878688.761398:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878688.761406:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878688.761408:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878688.761414:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1788878688.761418:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878688.761420:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878688.761430:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878688.761436:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878688.761438:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878688.761447:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878688.761451:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878688.761456:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878688.761460:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878688.761466:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:2.0:1788878688.761474:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290088320:192.168.203.129@tcp:250:kworker.0 00000008:00100000:3.0:1788878688.761475:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878688.761494:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.761496:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788878688.761502:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878688.761508:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878688.761515:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878688.761516:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878688.761522:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878688.761525:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878688.761529:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878688.761535:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878688.761540:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878688.761540:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878688.761558:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878688.761572:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290088448/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878688.761578:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878688.761585:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:3.0:1788878688.761591:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878688.761598:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878688.761600:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878688.761611:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878688.761615:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878688.761620:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878688.761623:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878688.761627:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878688.761633:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878688.761638:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9627 00000100:00080000:3.0:1788878688.761647:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878688.761651:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878688.761654:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878688.761660:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878688.761663:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878688.761667:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878688.761671:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878688.761680:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878688.761682:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:2.0:1788878688.761690:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1788878688.761691:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878688.761703:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878688.761707:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878688.761713:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878688.761718:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.761722:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878688.761723:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878688.761732:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878688.761735:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00000001:3.0:1788878688.761739:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.761743:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878688.761749:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.761752:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788878688.761766:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878688.761767:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878688.761773:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.761780:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304300585 00000100:00000001:3.0:1788878688.761782:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878688.761786:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878688.761792:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878688.761805:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.761807:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00100000:3.0:1788878688.761819:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290088576/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878688.761835:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878688.761838:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878688.761843:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.761849:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304300585 00000100:00080000:3.0:1788878688.761849:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878688.761863:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878688.761866:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878688.761867:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878688.761878:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878688.761882:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878688.761888:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788878688.761891:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878688.761955:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9627 00000100:00080000:3.0:1788878688.761965:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878688.761966:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788878688.761975:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878688.761979:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:2.0:1788878688.761994:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878688/real 1788878688] req@ffff94e50449f100 x1875765290088320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788878688.761994:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878688.762003:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878688.762007:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878688.762018:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000400:00000001:2.0:1788878688.762058:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878688.762062:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878688.762070:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878688.762074:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878688.762077:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.762092:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878688.762096:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878688.762105:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290088320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878688.762117:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.762127:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878688.762131:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.762134:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878688.762136:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.762140:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878688.762143:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878688.762149:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878688.762155:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878688.762159:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878688.762165:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290088320:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878688.762172:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.762175:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878688.762177:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878688.762181:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878688.762186:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878688.762190:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878688.762193:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878688.762200:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878688.762203:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878688.762208:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878688.762210:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.762213:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.762225:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878688.762229:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.762232:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878688.762235:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878688.762237:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878688.762239:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878688.762241:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878688.762254:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878688.762258:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878688.762261:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878688.762266:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290088448:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878688.762271:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878688.762275:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878688.762278:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878688.762280:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878688.762285:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.762295:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878688.762298:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:2.0:1788878688.762305:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878688.762342:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878688.762349:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878688.762353:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878688.762356:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.762359:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878688.762363:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.762371:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304300044 00000100:00000001:2.0:1788878688.762375:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304300585 00000100:00000001:2.0:1788878688.762378:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304300044 00000100:00000001:2.0:1788878688.762382:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304300585 00000100:00000001:2.0:1788878688.762385:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878688.762388:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.762391:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878688.762397:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878688/real 1788878688] req@ffff94e50449e300 x1875765290088448/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878688.762421:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878688.762423:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878688.762428:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878688.762430:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878688.762432:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.762435:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878688.762438:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878688.762444:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290088448/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878688.762455:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.762459:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878688.762463:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.762466:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878688.762468:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.762471:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878688.762497:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878688.762502:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878688.762509:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878688.762515:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878688.762530:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290088448:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878688.762530:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878688.762538:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878688.762551:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1788878688.762555:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878688.762558:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878688.762563:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878688.762567:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878688.762570:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878688.762574:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878688.762586:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878688.762590:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290088576:192.168.203.129@tcp:8:kworker.0 02000000:00000001:3.0:1788878688.762594:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878688.762603:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878688.762604:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878688.762609:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878688.762611:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878688.762616:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878688.762619:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:2.0:1788878688.762624:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:3.0:1788878688.762627:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788878688.762633:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788878688.762635:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878688.762642:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.762648:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788878688.762653:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878688.762654:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 02000000:00000001:3.0:1788878688.762660:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.762667:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878688.762668:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878688.762673:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878688.762677:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878688.762691:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788878688.762703:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1788878688.762712:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290088704/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878688.762716:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878688.762722:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878688.762727:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.762736:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878688.762743:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878688.762746:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878688.762754:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.762759:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304300042 00000100:00000001:3.0:1788878688.762766:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.762768:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304300585 00000100:00000001:2.0:1788878688.762776:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304300042 00000100:00000001:2.0:1788878688.762780:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304300585 00000100:00000001:2.0:1788878688.762785:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878688.762788:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.762792:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878688.762802:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878688/real 1788878688] req@ffff94e50449ca80 x1875765290088576/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878688.762817:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878688.762820:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878688.762826:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878688.762830:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878688.762834:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.762837:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878688.762839:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878688.762847:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290088576/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878688.762858:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.762861:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878688.762866:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.762868:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878688.762870:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.762873:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878688.762894:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878688.762898:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878688.762940:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878688.762945:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878688.762949:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290088576:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878688.762955:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.762958:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878688.762960:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878688.762979:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878688.762984:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878688.762986:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878688.762989:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878688.762993:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878688.762995:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878688.762998:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878688.763000:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.763003:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878688.763005:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878688.763007:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878688.763009:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878688.763011:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878688.763013:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878688.763016:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878688.763017:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878688.763020:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878688.763022:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.763025:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.763069:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878688.763073:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.763076:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878688.763078:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878688.763081:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878688.763083:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878688.763086:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878688.763090:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878688.763093:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878688.763096:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878688.763117:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290088704:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878688.763121:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878688.763123:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878688.763125:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878688.763127:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878688.763131:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.763139:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878688.763140:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:2.0:1788878688.763145:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878688.763161:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878688.763167:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878688.763170:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878688.763172:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.763175:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878688.763178:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.763183:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304300043 00000100:00000001:2.0:1788878688.763186:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304300585 00000100:00000001:2.0:1788878688.763189:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304300043 00000100:00000001:2.0:1788878688.763192:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304300585 00000100:00000001:2.0:1788878688.763194:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878688.763197:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.763199:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878688.763205:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878688/real 1788878688] req@ffff94e50449c000 x1875765290088704/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878688.763214:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878688.763216:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878688.763220:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878688.763222:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878688.763224:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.763226:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878688.763228:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878688.763233:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290088704/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878688.763243:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.763246:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878688.763249:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.763251:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878688.763253:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.763255:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878688.763272:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878688.763276:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878688.763281:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878688.763284:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878688.763288:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290088704:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878688.763293:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.763296:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878688.763298:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878688.763300:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878688.763303:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878688.763305:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878688.763306:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878688.763309:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878688.763311:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878688.763313:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878688.763315:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.763317:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878688.763321:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878688.763323:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878688.763325:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878688.763327:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878689.529084:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878689.529095:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878689.529103:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878689.529105:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878689.529134:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878689.529137:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878689.529141:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878690.553042:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878690.553049:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878690.553054:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878690.553056:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878690.553081:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878690.553084:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878690.553087:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878691.577000:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878691.577006:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878691.577022:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878691.577026:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878691.577029:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878691.577033:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878691.577036:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878691.577039:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878691.577041:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878691.577044:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878691.577129:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878691.577134:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878691.577138:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878692.601103:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878692.601111:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878692.601117:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878692.601118:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878692.601145:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878692.601148:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878692.601152:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878693.625066:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878693.625078:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878693.625082:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878693.625086:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878693.625094:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878693.625098:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878693.625101:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878693.625106:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878693.625112:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878693.625116:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878693.625156:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878693.625161:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878693.625166:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878693.881047:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878693.881061:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878693.881064:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878693.881069:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878693.881075:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878693.881085:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9632 00000100:00080000:3.0:1788878693.881098:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878693.881102:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878693.881105:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878693.881117:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878693.881121:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878693.881125:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878693.881128:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878693.881130:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878693.881132:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878693.881137:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878693.881140:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878693.881144:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878693.881147:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878693.881151:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878693.881153:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878693.881156:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878693.881168:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878693.881170:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878693.881174:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878693.881179:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878693.881184:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878693.881200:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878693.881211:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290088832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878693.881222:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878693.881227:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878693.881251:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878693.881254:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878693.881262:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878693.881264:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878693.881266:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878693.881269:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878693.881272:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9632 00000100:00080000:3.0:1788878693.881275:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878693.881277:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878693.881279:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878693.881282:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878693.881283:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878693.881286:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878693.881288:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878693.881291:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878693.881292:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878693.881296:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878693.881298:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878693.881300:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878693.881306:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878693.881308:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878693.881310:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878693.881312:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:3.0:1788878693.881316:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878693.881319:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00000001:3.0:1788878693.881320:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.881325:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878693.881325:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.881330:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788878693.881333:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878693.881334:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878693.881337:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878693.881341:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878693.881342:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878693.881343:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878693.881348:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878693.881349:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878693.881355:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878693.881359:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878693.881362:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290088832:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1788878693.881385:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290088960/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878693.881387:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878693.881396:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878693.881399:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1788878693.881401:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878693.881404:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878693.881407:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878693.881413:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878693.881415:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878693.881420:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878693.881425:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878693.881426:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878693.881428:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878693.881431:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878693.881435:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9632 00000100:00080000:3.0:1788878693.881439:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878693.881441:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878693.881443:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878693.881444:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878693.881448:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000020:00000001:3.0:1788878693.881451:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878693.881455:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878693.881457:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1788878693.881458:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878693.881463:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878693.881465:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878693.881468:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878693.881472:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878693.881476:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878693.881479:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878693.881482:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878693.881486:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788878693.881519:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878693.881536:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878693.881541:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878693.881544:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.881548:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878693.881555:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.881561:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304305705 00000100:00000001:3.0:1788878693.881576:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878693.881580:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878693.881583:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00000001:3.0:1788878693.881584:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878693.881591:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.881597:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304305705 02000000:00000001:3.0:1788878693.881598:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878693.881603:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878693.881606:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878693.881609:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1788878693.881610:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878693.881616:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.881618:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878693.881623:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878693.881626:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.881627:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878693.881639:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878693/real 1788878693] req@ffff94e50449f800 x1875765290088832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878693.881640:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290089088/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878693.881685:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878693.881691:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878693.881694:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788878693.881695:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878693.881700:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878693.881707:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:3.0:1788878693.881709:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878693.881713:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00080000:3.0:1788878693.881715:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878693.881720:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878693.881721:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788878693.881723:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878693.881731:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878693.881731:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878693.881738:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9632 00000100:00000001:0.0:1788878693.881740:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788878693.881743:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878693.881748:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878693.881751:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:0.0:1788878693.881753:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290088832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788878693.881756:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878693.881761:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878693.881769:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.881774:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878693.881778:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.881779:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878693.881781:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.881784:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878693.881786:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878693.881789:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878693.881794:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878693.881797:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878693.881801:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290088832:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878693.881807:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.881810:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878693.881812:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878693.881815:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878693.881820:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878693.881822:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878693.881825:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878693.881829:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878693.881831:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878693.881834:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878693.881836:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.881839:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.881853:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878693.881856:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.881860:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878693.881879:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878693.881880:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878693.881882:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878693.881884:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878693.881887:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878693.881890:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878693.881893:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878693.881899:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290088960:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878693.881905:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878693.881907:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878693.881909:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878693.881912:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1788878693.881947:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878693.881952:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878693.881955:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878693.881957:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:0.0:1788878693.881976:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.881990:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878693.881993:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00080000:3.0:1788878693.882016:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878693.882018:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878693.882024:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878693.882026:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878693.882031:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878693.882033:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878693.882035:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878693.882055:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878693.882058:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878693.882061:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878693.882068:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878693.882070:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788878693.882075:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878693.882076:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878693.882081:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878693.882085:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.882090:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878693.882094:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.882097:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878693.882101:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.882106:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:3.0:1788878693.882107:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290089216/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878693.882113:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878693.882122:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878693.882126:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304305164 00000100:00000001:3.0:1788878693.882127:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.882135:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304305705 00000100:00000001:3.0:1788878693.882136:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.882142:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304305164 00000100:00000001:0.0:1788878693.882148:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304305705 00000100:00000001:0.0:1788878693.882150:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878693.882153:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.882155:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878693.882162:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878693/real 1788878693] req@ffff94e50449c700 x1875765290088960/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878693.882180:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878693.882181:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878693.882184:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878693.882187:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878693.882188:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882190:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878693.882192:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878693.882195:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290088960/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878693.882203:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882205:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878693.882208:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882210:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878693.882211:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882213:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878693.882231:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878693.882234:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878693.882237:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878693.882239:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878693.882242:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290088960:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878693.882246:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878693.882247:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878693.882249:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878693.882252:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878693.882271:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878693.882272:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878693.882275:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290089088:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878693.882278:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878693.882280:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878693.882282:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878693.882283:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878693.882286:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.882292:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878693.882294:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:0.0:1788878693.882298:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878693.882308:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878693.882312:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878693.882313:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878693.882315:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.882316:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878693.882318:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.882321:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304305162 00000100:00000001:0.0:1788878693.882323:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304305705 00000100:00000001:0.0:1788878693.882326:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304305162 00000100:00000001:0.0:1788878693.882328:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304305705 00000100:00000001:0.0:1788878693.882330:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878693.882331:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.882333:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878693.882336:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878693/real 1788878693] req@ffff94e50449f480 x1875765290089088/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878693.882341:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878693.882342:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878693.882344:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878693.882346:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878693.882347:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882348:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878693.882350:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878693.882353:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290089088/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878693.882358:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882359:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878693.882362:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882363:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878693.882365:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882366:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878693.882382:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878693.882385:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878693.882388:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878693.882389:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878693.882392:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290089088:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878693.882395:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882396:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878693.882397:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878693.882399:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878693.882400:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878693.882401:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878693.882402:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878693.882404:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878693.882405:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878693.882407:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878693.882408:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882410:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878693.882411:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878693.882412:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878693.882413:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878693.882414:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878693.882415:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878693.882416:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878693.882417:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878693.882418:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878693.882419:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882420:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882423:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878693.882425:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.882426:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878693.882427:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878693.882428:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878693.882445:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878693.882447:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878693.882449:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878693.882451:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878693.882453:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878693.882456:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290089216:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878693.882458:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878693.882460:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878693.882461:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878693.882463:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878693.882465:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.882470:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878693.882472:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:0.0:1788878693.882476:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878693.882486:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878693.882490:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878693.882492:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878693.882493:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.882495:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878693.882498:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.882501:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304305162 00000100:00000001:0.0:1788878693.882503:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304305705 00000100:00000001:0.0:1788878693.882506:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304305162 00000100:00000001:0.0:1788878693.882508:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304305705 00000100:00000001:0.0:1788878693.882510:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878693.882512:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.882513:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878693.882517:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878693/real 1788878693] req@ffff94e50449d180 x1875765290089216/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878693.882523:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878693.882524:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878693.882526:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878693.882528:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878693.882529:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882531:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878693.882533:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878693.882536:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290089216/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878693.882543:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882545:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878693.882547:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882549:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878693.882550:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882551:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878693.882568:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878693.882570:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878693.882573:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878693.882574:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878693.882577:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290089216:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878693.882580:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882582:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878693.882583:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878693.882584:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878693.882586:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878693.882587:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878693.882589:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878693.882591:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878693.882592:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878693.882594:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878693.882595:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882597:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878693.882599:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878693.882601:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878693.882603:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878693.882605:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878694.649011:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878694.649019:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878694.649026:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878694.649028:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878694.649054:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878694.649056:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878694.649059:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878695.673001:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878695.673009:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878695.673018:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878695.673019:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878695.673088:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878695.673092:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878695.673095:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878696.697088:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878696.697095:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878696.697101:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878696.697102:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878696.697126:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878696.697129:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878696.697131:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878697.721003:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878697.721012:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878697.721019:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878697.721022:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878697.721053:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878697.721057:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878697.721062:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878698.745070:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878698.745078:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878698.745083:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878698.745085:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878698.745137:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878698.745140:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878698.745144:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878699.001017:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878699.001025:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878699.001028:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878699.001031:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878699.001034:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878699.001040:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9637 00000100:00080000:2.0:1788878699.001049:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878699.001051:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878699.001053:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878699.001060:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878699.001062:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878699.001065:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878699.001067:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878699.001068:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878699.001069:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878699.001078:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878699.001080:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878699.001082:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878699.001085:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878699.001087:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878699.001089:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878699.001091:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878699.001101:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878699.001102:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878699.001104:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878699.001106:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878699.001110:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878699.001122:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878699.001130:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290089344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878699.001138:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878699.001142:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878699.001153:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878699.001156:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878699.001160:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878699.001161:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878699.001163:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878699.001165:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878699.001167:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9637 00000100:00080000:2.0:1788878699.001169:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878699.001170:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878699.001171:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878699.001173:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878699.001173:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878699.001175:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878699.001176:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878699.001178:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878699.001179:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878699.001181:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878699.001183:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878699.001184:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878699.001188:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878699.001190:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878699.001191:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878699.001194:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878699.001196:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878699.001198:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878699.001201:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878699.001202:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878699.001203:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878699.001204:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788878699.001205:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878699.001207:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001211:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878699.001212:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001216:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1788878699.001218:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290089472/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878699.001221:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878699.001226:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878699.001235:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788878699.001236:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878699.001240:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878699.001241:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878699.001243:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878699.001248:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878699.001248:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878699.001254:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878699.001255:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878699.001257:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1788878699.001260:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290089344:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878699.001260:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878699.001264:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9637 00000100:00080000:2.0:1788878699.001270:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878699.001272:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788878699.001272:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878699.001275:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:0.0:1788878699.001278:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:2.0:1788878699.001278:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878699.001282:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1788878699.001283:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878699.001285:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 02000000:00000001:0.0:1788878699.001289:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1788878699.001290:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878699.001294:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878699.001294:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:0.0:1788878699.001297:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878699.001297:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878699.001298:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878699.001303:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878699.001306:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878699.001307:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878699.001308:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878699.001310:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878699.001312:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878699.001313:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878699.001316:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878699.001317:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878699.001318:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878699.001320:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788878699.001320:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878699.001325:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 02000000:00000001:2.0:1788878699.001326:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878699.001332:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001333:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1788878699.001338:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290089600/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878699.001344:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878699.001345:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878699.001347:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878699.001349:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878699.001351:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878699.001352:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878699.001353:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878699.001355:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878699.001357:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9637 00000100:00080000:2.0:1788878699.001359:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878699.001360:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878699.001361:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878699.001364:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878699.001365:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878699.001366:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878699.001368:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878699.001369:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878699.001370:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878699.001372:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878699.001374:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878699.001376:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:0.0:1788878699.001380:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:2.0:1788878699.001381:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878699.001386:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878699.001387:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878699.001389:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878699.001391:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878699.001392:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878699.001396:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878699.001397:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001398:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878699.001399:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878699.001402:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878699.001403:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001406:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878699.001407:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001411:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878699.001413:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290089728/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878699.001417:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878699.001422:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878699.001423:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001425:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878699.001426:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001434:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304310825 00000100:00000001:0.0:1788878699.001446:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878699.001454:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304310825 00000100:00000001:0.0:1788878699.001460:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878699.001463:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878699.001466:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001469:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878699.001476:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878699/real 1788878699] req@ffff94e5045e6300 x1875765290089344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878699.001489:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878699.001491:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878699.001495:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878699.001498:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878699.001500:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.001503:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878699.001505:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878699.001510:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290089344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878699.001520:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.001534:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878699.001537:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.001540:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878699.001541:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.001544:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878699.001546:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878699.001550:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878699.001555:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878699.001558:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878699.001563:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290089344:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878699.001568:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.001571:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878699.001573:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878699.001576:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878699.001579:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878699.001581:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878699.001583:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878699.001587:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878699.001589:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878699.001592:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878699.001593:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.001596:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.001603:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878699.001605:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001608:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878699.001609:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878699.001611:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878699.001613:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878699.001614:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878699.001617:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878699.001619:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878699.001622:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878699.001626:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290089472:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878699.001629:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878699.001632:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878699.001634:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878699.001636:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878699.001639:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001645:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878699.001647:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000100:00000001:0.0:1788878699.001652:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878699.001665:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878699.001670:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878699.001673:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878699.001674:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001677:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878699.001679:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001684:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304310283 00000100:00000001:0.0:1788878699.001688:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304310825 00000100:00000001:0.0:1788878699.001690:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304310283 00000100:00000001:0.0:1788878699.001694:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304310825 00000100:00000001:0.0:1788878699.001697:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878699.001700:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001702:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878699.001708:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878699/real 1788878699] req@ffff94e5045e7800 x1875765290089472/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878699.001718:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878699.001720:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878699.001724:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878699.001726:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878699.001727:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.001729:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878699.001731:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878699.001735:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290089472/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878699.001760:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.001764:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878699.001768:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.001770:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878699.001772:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.001775:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878699.001798:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878699.001803:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878699.001808:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878699.001812:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878699.001817:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290089472:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878699.001823:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878699.001825:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878699.001827:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878699.001830:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878699.001833:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878699.001836:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878699.001840:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290089600:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878699.001845:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878699.001848:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878699.001850:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878699.001852:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878699.001856:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001864:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878699.001866:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:0.0:1788878699.001871:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878699.001888:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878699.001895:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878699.001898:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878699.001901:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001903:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878699.001907:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001912:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304310281 00000100:00000001:0.0:1788878699.001915:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304310825 00000100:00000001:0.0:1788878699.001975:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304310282 00000100:00000001:0.0:1788878699.001980:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304310825 00000100:00000001:0.0:1788878699.001983:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878699.001986:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.001989:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878699.001995:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878699/real 1788878699] req@ffff94e5045e7100 x1875765290089600/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878699.002022:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878699.002023:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878699.002027:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878699.002029:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878699.002030:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.002032:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878699.002033:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878699.002037:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290089600/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878699.002044:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.002046:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878699.002049:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.002051:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878699.002052:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.002054:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878699.002066:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878699.002069:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878699.002072:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878699.002074:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878699.002077:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290089600:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878699.002081:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878699.002083:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878699.002085:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878699.002087:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878699.002089:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878699.002091:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878699.002094:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290089728:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878699.002097:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878699.002099:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878699.002101:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878699.002102:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878699.002106:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.002111:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878699.002112:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:0.0:1788878699.002116:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878699.002128:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878699.002132:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878699.002135:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878699.002136:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.002138:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878699.002141:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.002144:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304310282 00000100:00000001:0.0:1788878699.002146:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304310825 00000100:00000001:0.0:1788878699.002149:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304310282 00000100:00000001:0.0:1788878699.002152:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304310825 00000100:00000001:0.0:1788878699.002154:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878699.002156:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.002158:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878699.002162:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878699/real 1788878699] req@ffff94e5045e4a80 x1875765290089728/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878699.002170:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878699.002171:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878699.002174:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878699.002176:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878699.002177:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.002180:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878699.002181:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878699.002185:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290089728/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878699.002192:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.002194:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878699.002197:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.002198:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878699.002200:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.002202:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878699.002214:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878699.002217:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878699.002220:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878699.002223:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878699.002226:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290089728:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878699.002230:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.002232:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878699.002234:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878699.002235:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878699.002238:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878699.002239:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878699.002241:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878699.002243:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878699.002244:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878699.002246:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878699.002247:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.002249:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878699.002250:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878699.002252:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878699.002254:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878699.002255:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878699.002257:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878699.002258:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878699.002259:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878699.002261:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878699.002262:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.002264:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878699.002265:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878699.002266:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878699.002268:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878699.002269:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878699.002271:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878699.002273:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878699.002274:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878699.002276:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878699.002277:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.002279:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878699.002283:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878699.002284:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878699.002286:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878699.002287:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878699.769047:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878699.769052:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878699.769057:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878699.769058:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878699.769109:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878699.769113:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878699.769117:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878700.793009:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878700.793014:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878700.793019:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878700.793020:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878700.793051:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878700.793054:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878700.793056:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878701.816979:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878701.817002:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878701.817007:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878701.817011:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878701.817013:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878701.817017:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878701.817019:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878701.817020:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878701.817022:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878701.817023:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878701.817038:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878701.817040:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878701.817042:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878702.841112:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878702.841122:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878702.841129:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878702.841131:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878702.841160:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878702.841165:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878702.841169:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878703.865104:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878703.865114:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878703.865116:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878703.865120:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878703.865126:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878703.865128:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878703.865131:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878703.865134:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878703.865152:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878703.865154:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878703.865224:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878703.865231:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878703.865237:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878704.121074:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878704.121085:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878704.121087:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878704.121091:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878704.121096:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878704.121103:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9642 00000100:00080000:0.0:1788878704.121114:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878704.121117:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878704.121119:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878704.121127:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878704.121130:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878704.121133:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878704.121136:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878704.121138:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878704.121140:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878704.121144:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878704.121148:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878704.121151:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878704.121154:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878704.121157:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878704.121160:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878704.121162:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878704.121174:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878704.121176:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878704.121179:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878704.121182:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878704.121186:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878704.121198:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878704.121211:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897aa00 x1875765290089856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878704.121225:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878704.121231:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878704.121277:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878704.121284:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878704.121291:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878704.121294:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878704.121298:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878704.121303:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878704.121310:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9642 00000100:00080000:0.0:1788878704.121318:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878704.121323:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878704.121325:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878704.121329:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788878704.121331:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878704.121337:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788878704.121338:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788878704.121340:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878704.121344:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788878704.121347:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878704.121350:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878704.121353:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878704.121355:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878704.121358:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788878704.121365:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1788878704.121367:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:2.0:1788878704.121370:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878704.121373:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878704.121379:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878704.121386:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878704.121388:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878704.121389:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878704.121397:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:2.0:1788878704.121403:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290089856:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1788878704.121404:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878704.121416:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.121418:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788878704.121423:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878704.121437:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878704.121441:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878704.121442:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878704.121447:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878704.121449:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788878704.121454:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878704.121458:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878704.121459:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878704.121461:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878704.121475:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.121484:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00100000:0.0:1788878704.121485:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290089984/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878704.121498:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:0.0:1788878704.121500:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878704.121509:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.121510:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788878704.121519:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878704.121525:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878704.121529:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878704.121532:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878704.121535:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878704.121539:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878704.121542:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9642 00000100:00080000:0.0:1788878704.121563:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1788878704.121564:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878704.121573:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878704.121576:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878704.121581:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878704.121583:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788878704.121585:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:0.0:1788878704.121587:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878704.121595:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878704.121596:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878704.121603:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878704.121603:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.121608:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878704.121610:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878704.121620:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878704.121621:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878704.121630:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.121631:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304315945 02000000:00000001:0.0:1788878704.121633:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878704.121641:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878704.121644:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878704.121646:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878704.121648:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00100000:0.0:1788878704.121650:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878704.121658:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878704.121661:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.121661:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304315945 02000000:00000001:0.0:1788878704.121668:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878704.121671:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1788878704.121673:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878704.121680:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878704.121684:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878704.121684:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878704.121690:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.121695:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878704.121701:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.121703:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878704.121712:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978700 x1875765290090112/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788878704.121722:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878704/real 1788878704] req@ffff94e50897aa00 x1875765290089856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878704.121727:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878704.121733:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878704.121739:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878704.121747:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878704.121758:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878704.121763:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878704.121766:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:2.0:1788878704.121768:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788878704.121781:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000400:00000001:2.0:1788878704.121785:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:0.0:1788878704.121791:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9642 00000100:00080000:0.0:1788878704.121798:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878704.121814:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878704.121819:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878704.121823:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878704.121824:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878704.121830:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788878704.121831:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878704.121834:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878704.121840:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878704.121840:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878704.121848:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788878704.121849:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878704.121858:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878704.121859:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:2.0:1788878704.121866:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290089856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878704.121867:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878704.121870:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878704.121872:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878704.121874:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878704.121876:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878704.121881:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878704.121885:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878704.121891:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878704.121897:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878704.121905:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878704.121909:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788878704.121910:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.121974:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878704.121980:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.121982:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878704.121984:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.121986:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878704.121989:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878704.121993:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:2.0:1788878704.121995:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:0.0:1788878704.121998:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878704.122003:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878704.122004:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878704.122010:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1788878704.122014:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878704.122016:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290089856:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878704.122025:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878704.122026:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290090240/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878704.122030:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878704.122034:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878704.122039:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1788878704.122045:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788878704.122045:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878704.122049:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:0.0:1788878704.122051:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878704.122053:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:0.0:1788878704.122056:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878704.122059:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878704.122062:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878704.122067:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878704.122068:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122071:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122080:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878704.122082:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.122085:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878704.122087:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878704.122089:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878704.122090:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878704.122092:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878704.122094:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878704.122096:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878704.122098:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878704.122103:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290089984:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878704.122106:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878704.122109:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878704.122111:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878704.122113:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878704.122116:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.122123:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878704.122124:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:2.0:1788878704.122129:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878704.122155:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878704.122160:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878704.122163:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878704.122164:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.122166:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878704.122169:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.122173:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304315404 00000100:00000001:2.0:1788878704.122176:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304315945 00000100:00000001:2.0:1788878704.122179:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304315404 00000100:00000001:2.0:1788878704.122181:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304315945 00000100:00000001:2.0:1788878704.122183:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878704.122185:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.122187:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878704.122192:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878704/real 1788878704] req@ffff94e50897a300 x1875765290089984/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878704.122200:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878704.122202:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878704.122204:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878704.122206:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878704.122208:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122209:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878704.122211:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878704.122215:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290089984/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878704.122222:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122224:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878704.122243:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122245:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878704.122247:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122249:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878704.122262:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878704.122265:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878704.122269:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878704.122272:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878704.122275:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290089984:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878704.122280:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878704.122282:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878704.122284:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878704.122287:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878704.122289:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878704.122292:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878704.122297:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290090112:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878704.122302:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878704.122305:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878704.122308:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878704.122311:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878704.122316:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.122324:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878704.122325:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00000001:2.0:1788878704.122330:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878704.122351:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878704.122356:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878704.122359:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878704.122361:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.122363:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878704.122366:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.122370:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304315402 00000100:00000001:2.0:1788878704.122373:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304315945 00000100:00000001:2.0:1788878704.122376:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304315402 00000100:00000001:2.0:1788878704.122378:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304315945 00000100:00000001:2.0:1788878704.122381:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878704.122383:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.122386:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878704.122390:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878704/real 1788878704] req@ffff94e508978700 x1875765290090112/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878704.122400:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878704.122401:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878704.122404:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878704.122406:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878704.122408:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122411:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878704.122413:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878704.122417:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290090112/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878704.122436:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122438:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878704.122441:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122442:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878704.122444:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122446:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878704.122457:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878704.122459:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878704.122463:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878704.122465:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878704.122468:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290090112:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878704.122472:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878704.122474:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878704.122475:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878704.122478:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878704.122480:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878704.122484:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878704.122490:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290090240:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878704.122495:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878704.122498:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878704.122502:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878704.122505:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878704.122511:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.122520:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878704.122522:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:2.0:1788878704.122528:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878704.122552:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878704.122561:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878704.122565:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878704.122568:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.122571:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878704.122576:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.122583:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304315402 00000100:00000001:2.0:1788878704.122587:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304315945 00000100:00000001:2.0:1788878704.122591:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304315402 00000100:00000001:2.0:1788878704.122595:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304315945 00000100:00000001:2.0:1788878704.122599:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878704.122602:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.122605:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878704.122614:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878704/real 1788878704] req@ffff94e508978a80 x1875765290090240/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878704.122630:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878704.122633:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878704.122638:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878704.122641:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878704.122644:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122647:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878704.122649:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878704.122656:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290090240/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878704.122670:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122674:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878704.122679:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122682:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878704.122685:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122688:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878704.122707:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878704.122712:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878704.122719:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878704.122723:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878704.122730:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290090240:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878704.122737:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122740:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878704.122742:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878704.122745:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878704.122750:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878704.122753:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878704.122755:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878704.122760:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878704.122762:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878704.122766:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878704.122768:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122772:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878704.122786:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878704.122788:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878704.122791:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878704.122794:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878704.122796:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878704.122799:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878704.122801:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878704.122804:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878704.122806:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122809:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878704.122828:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878704.122830:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878704.122833:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878704.122834:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878704.122836:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878704.122839:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878704.122840:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878704.122843:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878704.122845:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122847:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878704.122852:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878704.122854:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878704.122857:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878704.122860:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878704.888997:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878704.889002:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878704.889006:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878704.889007:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878704.889023:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878704.889025:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878704.889027:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878705.913010:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878705.913015:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878705.913018:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878705.913019:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878705.913036:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878705.913038:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878705.913040:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878706.937015:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878706.937022:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878706.937028:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878706.937029:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878706.937052:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878706.937055:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878706.937068:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878707.961037:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878707.961045:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878707.961050:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878707.961051:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878707.961135:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878707.961139:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878707.961143:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878708.985075:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878708.985086:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878708.985093:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878708.985095:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878708.985129:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878708.985133:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878708.985138:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878709.241099:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878709.241114:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878709.241118:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878709.241124:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878709.241130:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878709.241140:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9648 00000100:00080000:2.0:1788878709.241157:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878709.241160:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878709.241163:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878709.241175:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878709.241179:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878709.241184:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878709.241187:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878709.241190:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878709.241191:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878709.241197:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878709.241199:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878709.241203:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878709.241208:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878709.241211:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241214:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878709.241217:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878709.241235:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878709.241237:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241242:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878709.241246:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878709.241253:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241270:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878709.241282:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290090368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878709.241296:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878709.241302:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241330:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878709.241334:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878709.241338:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878709.241340:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878709.241342:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878709.241346:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878709.241350:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9648 00000100:00080000:2.0:1788878709.241354:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878709.241357:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878709.241359:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878709.241364:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878709.241366:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878709.241368:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878709.241373:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788878709.241373:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878709.241381:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878709.241384:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878709.241384:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878709.241391:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1788878709.241391:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878709.241398:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878709.241399:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.241406:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878709.241407:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878709.241417:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878709.241418:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878709.241422:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878709.241424:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878709.241430:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241431:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878709.241439:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878709.241444:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290090368:192.168.203.129@tcp:250:kworker.0 00000008:00000001:2.0:1788878709.241446:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241452:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.241458:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878709.241460:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878709.241465:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241467:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878709.241469:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878709.241470:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878709.241476:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878709.241479:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241483:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878709.241488:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1788878709.241495:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290090496/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788878709.241503:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241507:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878709.241512:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241515:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878709.241518:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878709.241522:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878709.241523:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878709.241526:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878709.241529:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878709.241532:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9648 00000100:00000001:0.0:1788878709.241536:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788878709.241537:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878709.241543:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878709.241544:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 00000100:00000001:2.0:1788878709.241546:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878709.241553:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878709.241556:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878709.241559:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878709.241562:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1788878709.241562:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878709.241568:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878709.241570:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878709.241574:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878709.241576:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878709.241578:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878709.241581:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878709.241583:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241585:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878709.241588:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878709.241591:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241593:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878709.241598:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878709.241600:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241602:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878709.241604:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878709.241607:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241614:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878709.241618:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290090624/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878709.241629:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878709.241636:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878709.241641:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241644:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878709.241647:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878709.241652:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878709.241654:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878709.241655:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788878709.241657:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878709.241663:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878709.241665:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788878709.241671:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9648 00000100:00000001:0.0:1788878709.241672:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878709.241678:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878709.241679:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878709.241685:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878709.241688:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878709.241690:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878709.241692:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878709.241697:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878709.241699:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304321065 00000020:00000001:2.0:1788878709.241700:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878709.241709:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878709.241713:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878709.241714:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878709.241718:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878709.241720:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878709.241722:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878709.241725:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1788878709.241726:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878709.241770:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241776:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878709.241777:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304321065 00000008:00100000:2.0:1788878709.241780:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878709.241790:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000008:00000001:2.0:1788878709.241791:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241802:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.241806:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878709.241828:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878709.241833:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878709.241834:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.241843:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878709.241851:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878709.241858:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1788878709.241860:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878709/real 1788878709] req@ffff94e5045e5180 x1875765290090368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788878709.241862:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878709.241876:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878709.241947:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290090752/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1788878709.241949:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878709.241954:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878709.241962:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878709.241967:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878709.241972:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878709.241973:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.241980:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878709.241982:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.241985:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878709.241990:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878709.241999:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290090368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878709.242014:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242021:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878709.242027:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242032:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878709.242035:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242039:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878709.242044:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878709.242049:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878709.242057:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878709.242062:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878709.242069:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290090368:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878709.242078:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242083:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878709.242086:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878709.242090:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878709.242097:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878709.242099:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878709.242102:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878709.242107:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878709.242109:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878709.242113:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878709.242115:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242118:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242127:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878709.242130:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.242132:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878709.242134:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878709.242136:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878709.242138:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878709.242140:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878709.242143:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878709.242145:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878709.242148:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878709.242152:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290090496:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878709.242156:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878709.242159:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878709.242161:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878709.242163:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878709.242167:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.242176:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878709.242178:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:0.0:1788878709.242183:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878709.242200:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878709.242206:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878709.242209:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878709.242210:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.242213:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878709.242216:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.242221:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304320524 00000100:00000001:0.0:1788878709.242225:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304321065 00000100:00000001:0.0:1788878709.242227:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304320524 00000100:00000001:0.0:1788878709.242230:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304321065 00000100:00000001:0.0:1788878709.242232:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878709.242235:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.242237:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878709.242242:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878709/real 1788878709] req@ffff94e5045e5880 x1875765290090496/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878709.242251:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878709.242253:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878709.242255:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878709.242257:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878709.242259:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242262:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878709.242263:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878709.242267:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290090496/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878709.242276:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242278:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878709.242281:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242283:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878709.242285:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242287:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878709.242306:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878709.242309:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878709.242313:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878709.242316:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878709.242319:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290090496:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878709.242323:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878709.242325:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878709.242326:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878709.242329:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878709.242330:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878709.242332:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878709.242336:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290090624:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878709.242339:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878709.242341:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878709.242343:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878709.242345:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878709.242348:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.242354:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878709.242355:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:0.0:1788878709.242359:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878709.242370:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878709.242374:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878709.242376:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878709.242378:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.242379:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878709.242382:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.242385:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304320522 00000100:00000001:0.0:1788878709.242388:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304321065 00000100:00000001:0.0:1788878709.242391:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304320522 00000100:00000001:0.0:1788878709.242393:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304321065 00000100:00000001:0.0:1788878709.242396:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878709.242398:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.242400:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878709.242404:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878709/real 1788878709] req@ffff94e5045e6a00 x1875765290090624/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878709.242413:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878709.242414:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878709.242416:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878709.242418:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878709.242419:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242421:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878709.242422:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878709.242426:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290090624/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878709.242434:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242436:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878709.242439:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242440:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878709.242442:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242443:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878709.242460:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878709.242462:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878709.242466:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878709.242468:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878709.242472:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290090624:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878709.242478:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878709.242480:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878709.242482:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878709.242486:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878709.242489:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878709.242493:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878709.242498:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290090752:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878709.242504:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878709.242506:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878709.242509:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878709.242512:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878709.242518:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.242527:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878709.242530:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:0.0:1788878709.242537:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878709.242557:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878709.242565:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878709.242568:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878709.242570:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.242572:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878709.242575:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.242579:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304320522 00000100:00000001:0.0:1788878709.242582:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304321065 00000100:00000001:0.0:1788878709.242585:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304320522 00000100:00000001:0.0:1788878709.242588:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304321065 00000100:00000001:0.0:1788878709.242590:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878709.242592:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.242594:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878709.242598:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878709/real 1788878709] req@ffff94e5045e6680 x1875765290090752/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878709.242607:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878709.242608:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878709.242611:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878709.242613:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878709.242614:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242616:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878709.242618:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878709.242622:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290090752/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878709.242638:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242641:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878709.242643:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242645:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878709.242646:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242648:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878709.242665:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878709.242668:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878709.242671:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878709.242673:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878709.242676:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290090752:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878709.242680:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242683:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878709.242684:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878709.242686:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878709.242689:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878709.242690:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878709.242692:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878709.242694:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878709.242696:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878709.242698:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878709.242700:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242702:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878709.242703:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878709.242704:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878709.242707:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878709.242708:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878709.242710:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878709.242712:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878709.242714:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878709.242715:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878709.242717:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242718:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878709.242719:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878709.242721:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878709.242723:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878709.242725:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878709.242726:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878709.242728:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878709.242729:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878709.242731:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878709.242732:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242734:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878709.242738:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878709.242740:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878709.242741:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878709.242743:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878710.009043:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878710.009055:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878710.009064:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878710.009065:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878710.009096:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878710.009099:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878710.009103:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878711.033059:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878711.033069:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878711.033076:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878711.033081:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878711.033084:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878711.033089:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878711.033092:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878711.033094:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878711.033097:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878711.033099:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878711.033129:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878711.033132:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878711.033136:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878712.057060:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878712.057076:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878712.057082:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878712.057083:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878712.057192:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878712.057202:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878712.057210:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878713.080994:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878713.081001:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878713.081005:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878713.081009:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878713.081015:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878713.081019:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878713.081021:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878713.081024:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878713.081029:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878713.081031:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878713.081056:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878713.081060:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878713.081064:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878714.104978:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878714.104985:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878714.104990:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878714.104992:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878714.105011:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878714.105014:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878714.105018:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878714.361117:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878714.361142:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878714.361146:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878714.361150:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878714.361157:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878714.361166:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9653 00000100:00080000:0.0:1788878714.361181:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878714.361185:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878714.361187:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878714.361198:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878714.361202:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878714.361206:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878714.361209:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878714.361211:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878714.361212:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878714.361218:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878714.361223:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878714.361228:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878714.361234:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878714.361239:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.361242:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878714.361253:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878714.361271:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878714.361273:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.361279:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878714.361282:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878714.361290:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.361317:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878714.361333:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290090880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878714.361348:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878714.361355:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.361378:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878714.361383:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878714.361388:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878714.361390:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878714.361392:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878714.361397:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878714.361401:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9653 00000100:00080000:0.0:1788878714.361408:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878714.361411:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878714.361413:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878714.361417:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878714.361420:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788878714.361420:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788878714.361424:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878714.361429:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788878714.361430:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878714.361436:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878714.361436:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878714.361442:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878714.361443:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878714.361450:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878714.361452:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:2.0:1788878714.361458:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788878714.361459:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878714.361464:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878714.361465:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878714.361472:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878714.361474:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878714.361482:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878714.361483:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290090880:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878714.361487:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878714.361494:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:0.0:1788878714.361497:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788878714.361501:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:0.0:1788878714.361504:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878714.361507:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.361509:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878714.361514:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878714.361519:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878714.361523:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878714.361525:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.361527:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878714.361532:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878714.361535:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.361543:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878714.361550:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290091008/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878714.361554:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878714.361559:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:2.0:1788878714.361564:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788878714.361570:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878714.361573:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.361577:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878714.361580:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878714.361583:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878714.361585:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878714.361587:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878714.361590:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878714.361593:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9653 00000100:00080000:0.0:1788878714.361597:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878714.361599:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878714.361600:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878714.361612:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878714.361613:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878714.361617:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878714.361619:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878714.361621:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878714.361639:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878714.361648:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:2.0:1788878714.361649:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878714.361651:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878714.361656:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878714.361660:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878714.361663:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.361666:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878714.361669:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878714.361673:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.361675:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878714.361676:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878714.361686:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878714.361687:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1788878714.361688:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.361694:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878714.361695:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878714.361696:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878714.361701:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.361705:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878714.361709:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878714.361718:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290091136/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878714.361719:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.361734:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878714.361735:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304326185 00000100:00000001:0.0:1788878714.361739:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.361746:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878714.361749:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878714.361754:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878714.361755:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878714.361757:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878714.361759:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878714.361761:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878714.361768:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9653 00000100:00080000:0.0:1788878714.361773:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878714.361777:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878714.361779:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878714.361779:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304326185 00000020:00000001:0.0:1788878714.361792:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788878714.361794:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:0.0:1788878714.361795:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878714.361803:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878714.361807:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788878714.361812:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878714.361815:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.361820:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878714.361824:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788878714.361826:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878714.361840:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1788878714.361842:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878714/real 1788878714] req@ffff94e508978e00 x1875765290090880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878714.361849:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878714.361874:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878714.361881:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:2.0:1788878714.361912:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878714.361915:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1788878714.361944:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.361947:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878714.361949:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878714.361952:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.361954:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.361958:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:0.0:1788878714.361959:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878714.361964:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.361967:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878714.361967:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:0.0:1788878714.361972:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878714.361973:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788878714.361975:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.361983:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878714.361986:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.361989:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878714.361995:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290091264/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788878714.362002:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290090880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878714.362009:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878714.362015:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878714.362020:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.362028:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362035:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878714.362040:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362043:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878714.362045:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362049:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878714.362052:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878714.362057:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878714.362063:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878714.362067:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878714.362072:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290090880:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878714.362079:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362083:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878714.362085:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878714.362089:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878714.362094:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878714.362096:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878714.362099:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878714.362105:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878714.362108:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878714.362112:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878714.362115:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362119:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362138:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878714.362142:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.362144:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878714.362147:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878714.362149:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878714.362151:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878714.362153:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878714.362155:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878714.362158:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878714.362161:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878714.362165:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290091008:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878714.362170:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878714.362173:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878714.362175:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878714.362178:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878714.362181:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.362189:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878714.362191:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:2.0:1788878714.362197:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878714.362214:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878714.362221:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878714.362224:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878714.362226:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.362228:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878714.362232:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.362238:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304325644 00000100:00000001:2.0:1788878714.362242:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304326185 00000100:00000001:2.0:1788878714.362255:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304325644 00000100:00000001:2.0:1788878714.362259:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304326185 00000100:00000001:2.0:1788878714.362262:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878714.362265:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.362267:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878714.362274:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878714/real 1788878714] req@ffff94e50897bb80 x1875765290091008/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878714.362285:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878714.362287:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878714.362291:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878714.362293:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878714.362295:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362298:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878714.362300:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878714.362306:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290091008/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878714.362316:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362319:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878714.362323:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362325:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878714.362327:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362330:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878714.362351:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878714.362356:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878714.362361:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878714.362364:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878714.362369:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290091008:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878714.362393:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878714.362395:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878714.362398:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878714.362402:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878714.362404:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878714.362407:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878714.362412:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290091136:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878714.362417:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878714.362420:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878714.362422:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878714.362426:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878714.362431:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.362441:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878714.362443:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:2.0:1788878714.362448:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878714.362470:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878714.362477:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878714.362480:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878714.362482:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.362485:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878714.362490:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.362495:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304325642 00000100:00000001:2.0:1788878714.362499:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304326185 00000100:00000001:2.0:1788878714.362503:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304325642 00000100:00000001:2.0:1788878714.362506:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304326185 00000100:00000001:2.0:1788878714.362509:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878714.362512:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.362515:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878714.362522:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878714/real 1788878714] req@ffff94e50897ad80 x1875765290091136/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878714.362534:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878714.362536:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878714.362541:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878714.362543:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878714.362546:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362549:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878714.362552:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878714.362558:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290091136/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878714.362569:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362573:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878714.362577:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362579:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878714.362581:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362584:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878714.362605:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878714.362609:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878714.362613:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878714.362617:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878714.362623:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290091136:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878714.362629:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878714.362631:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878714.362633:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878714.362636:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878714.362656:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878714.362659:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878714.362664:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290091264:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878714.362670:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878714.362673:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878714.362675:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878714.362678:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878714.362683:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.362691:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878714.362694:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:2.0:1788878714.362699:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878714.362718:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878714.362726:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878714.362729:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878714.362732:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.362734:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878714.362738:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.362743:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304325642 00000100:00000001:2.0:1788878714.362747:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304326185 00000100:00000001:2.0:1788878714.362751:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304325642 00000100:00000001:2.0:1788878714.362755:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304326185 00000100:00000001:2.0:1788878714.362758:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878714.362762:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.362766:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878714.362773:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878714/real 1788878714] req@ffff94e508979c00 x1875765290091264/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878714.362787:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878714.362789:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878714.362793:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878714.362796:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878714.362799:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362802:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878714.362804:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878714.362811:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290091264/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878714.362823:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362826:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878714.362830:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362833:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878714.362835:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362838:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878714.362883:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878714.362904:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878714.362910:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878714.362913:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878714.362944:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290091264:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878714.362952:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362954:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878714.362956:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878714.362958:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878714.362962:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878714.362964:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878714.362966:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878714.362970:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878714.362971:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878714.362976:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878714.362978:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.362980:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878714.362982:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878714.362984:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878714.362986:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878714.362988:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878714.362990:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878714.362992:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878714.362994:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878714.362997:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878714.362999:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.363002:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878714.363003:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878714.363005:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878714.363008:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878714.363010:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878714.363012:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878714.363014:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878714.363016:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878714.363018:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878714.363020:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.363023:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878714.363027:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878714.363030:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878714.363033:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878714.363035:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878715.128994:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878715.129001:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878715.129006:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878715.129007:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878715.129029:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878715.129033:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878715.129035:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878716.153081:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878716.153088:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878716.153094:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878716.153096:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878716.153177:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878716.153181:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878716.153184:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878717.177055:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878717.177062:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878717.177068:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878717.177070:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878717.177112:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878717.177115:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878717.177118:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878718.201042:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878718.201048:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878718.201054:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878718.201055:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878718.201103:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878718.201107:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878718.201112:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878719.225036:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878719.225044:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878719.225050:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878719.225052:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878719.225104:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878719.225108:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878719.225111:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878719.481071:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878719.481088:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878719.481103:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878719.481109:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878719.481116:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878719.481128:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9658 00000100:00080000:3.0:1788878719.481145:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878719.481151:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878719.481154:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878719.481166:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878719.481171:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878719.481176:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878719.481180:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878719.481183:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878719.481185:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878719.481193:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878719.481197:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878719.481202:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878719.481208:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878719.481213:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481216:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878719.481236:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878719.481255:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878719.481258:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481264:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878719.481269:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878719.481276:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481295:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878719.481310:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290091392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878719.481339:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878719.481347:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481372:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878719.481379:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878719.481384:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878719.481387:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878719.481390:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878719.481395:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878719.481400:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9658 00000100:00080000:3.0:1788878719.481406:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878719.481409:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878719.481411:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878719.481414:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878719.481416:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878719.481418:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878719.481421:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878719.481423:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878719.481425:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878719.481429:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878719.481431:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878719.481434:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878719.481443:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878719.481446:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481448:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878719.481455:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:3.0:1788878719.481455:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878719.481462:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.481463:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878719.481465:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.481469:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878719.481473:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788878719.481474:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878719.481476:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878719.481479:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878719.481484:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878719.481485:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878719.481490:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878719.481492:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878719.481494:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878719.481497:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481505:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878719.481507:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290091392:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878719.481517:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1788878719.481518:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290091520/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878719.481529:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878719.481532:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878719.481537:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878719.481537:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878719.481543:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878719.481544:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1788878719.481552:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788878719.481553:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878719.481558:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878719.481560:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1788878719.481562:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481564:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878719.481572:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9658 00000100:00080000:3.0:1788878719.481576:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878719.481578:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878719.481580:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878719.481583:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878719.481585:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878719.481587:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878719.481590:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878719.481591:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878719.481593:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878719.481597:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:3.0:1788878719.481598:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878719.481606:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788878719.481607:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878719.481613:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878719.481615:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878719.481618:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878719.481621:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481622:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878719.481625:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878719.481628:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481630:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878719.481636:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878719.481637:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481638:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878719.481640:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878719.481642:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481647:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878719.481652:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290091648/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878719.481659:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878719.481665:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788878719.481666:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878719.481669:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878719.481674:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878719.481678:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878719.481680:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878719.481681:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878719.481706:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878719.481710:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9658 00000100:00080000:3.0:1788878719.481713:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878719.481715:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878719.481716:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878719.481720:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878719.481721:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878719.481723:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878719.481724:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:3.0:1788878719.481726:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878719.481731:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878719.481733:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878719.481736:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1788878719.481737:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878719.481743:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481745:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.481761:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788878719.481763:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878719.481770:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878719.481773:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878719.481776:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481782:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878719.481784:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304331305 00000008:00100000:3.0:1788878719.481787:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878719.481795:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481797:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.481805:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788878719.481806:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878719.481843:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481848:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878719.481850:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878719.481853:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481859:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.481862:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304331305 00000100:00100000:3.0:1788878719.481868:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290091776/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878719.481886:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878719.481889:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878719.481892:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.481893:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878719.481950:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878719.481954:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.481958:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878719.481967:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878719/real 1788878719] req@ffff94e50449ce00 x1875765290091392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878719.481982:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878719.481984:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878719.481995:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878719.481999:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878719.482002:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482006:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878719.482008:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878719.482015:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290091392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878719.482023:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482035:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878719.482038:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482039:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878719.482041:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482043:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878719.482045:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878719.482049:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878719.482054:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878719.482056:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878719.482061:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290091392:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878719.482066:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482068:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878719.482069:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878719.482072:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878719.482076:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878719.482079:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878719.482081:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878719.482087:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878719.482099:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878719.482103:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878719.482104:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482107:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482115:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878719.482118:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.482120:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878719.482122:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878719.482123:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878719.482125:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878719.482127:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878719.482129:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878719.482131:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878719.482134:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878719.482137:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290091520:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878719.482140:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878719.482142:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878719.482144:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878719.482145:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878719.482148:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.482155:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878719.482156:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:2.0:1788878719.482162:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878719.482176:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878719.482181:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878719.482184:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878719.482185:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.482187:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878719.482189:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.482194:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304330764 00000100:00000001:2.0:1788878719.482197:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304331305 00000100:00000001:2.0:1788878719.482199:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304330764 00000100:00000001:2.0:1788878719.482202:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304331305 00000100:00000001:2.0:1788878719.482204:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878719.482206:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.482207:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878719.482211:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878719/real 1788878719] req@ffff94e50449d880 x1875765290091520/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878719.482219:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878719.482220:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878719.482222:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878719.482223:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878719.482226:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482227:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878719.482229:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878719.482232:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290091520/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878719.482239:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482241:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878719.482243:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482244:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878719.482245:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482247:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878719.482264:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878719.482266:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878719.482269:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878719.482271:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878719.482274:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290091520:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878719.482277:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878719.482279:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878719.482280:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878719.482282:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878719.482284:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878719.482286:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878719.482289:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290091648:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878719.482309:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878719.482311:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878719.482313:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878719.482314:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878719.482317:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.482324:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878719.482325:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:2.0:1788878719.482330:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878719.482342:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878719.482346:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878719.482348:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878719.482350:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.482352:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878719.482354:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.482358:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304330762 00000100:00000001:2.0:1788878719.482361:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304331305 00000100:00000001:2.0:1788878719.482363:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304330762 00000100:00000001:2.0:1788878719.482365:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304331305 00000100:00000001:2.0:1788878719.482368:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878719.482370:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.482372:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878719.482375:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878719/real 1788878719] req@ffff94e50449c380 x1875765290091648/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878719.482383:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878719.482384:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878719.482387:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878719.482388:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878719.482389:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482391:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878719.482392:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878719.482396:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290091648/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878719.482409:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482411:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878719.482414:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482416:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878719.482417:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482419:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878719.482435:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878719.482438:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878719.482441:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878719.482444:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878719.482447:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290091648:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878719.482451:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878719.482452:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878719.482454:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878719.482456:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878719.482458:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878719.482460:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878719.482462:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290091776:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878719.482465:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878719.482467:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878719.482468:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878719.482470:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878719.482479:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.482482:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878719.482484:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:2.0:1788878719.482487:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878719.482496:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878719.482500:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878719.482502:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878719.482503:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.482505:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878719.482508:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.482511:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304330762 00000100:00000001:2.0:1788878719.482514:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304331305 00000100:00000001:2.0:1788878719.482516:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304330762 00000100:00000001:2.0:1788878719.482519:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304331305 00000100:00000001:2.0:1788878719.482522:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878719.482524:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.482525:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878719.482530:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878719/real 1788878719] req@ffff94e50449fb80 x1875765290091776/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878719.482538:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878719.482540:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878719.482542:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878719.482543:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878719.482545:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482547:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878719.482548:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878719.482552:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290091776/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878719.482560:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482562:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878719.482564:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482565:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878719.482567:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482568:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878719.482584:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878719.482586:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878719.482589:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878719.482591:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878719.482594:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290091776:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878719.482598:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482600:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878719.482601:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878719.482603:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878719.482606:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878719.482607:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878719.482609:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878719.482611:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878719.482612:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878719.482614:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878719.482616:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482618:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878719.482620:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878719.482621:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878719.482623:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878719.482624:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878719.482626:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878719.482628:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878719.482629:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878719.482630:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878719.482631:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482633:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878719.482634:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878719.482635:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878719.482638:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878719.482639:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878719.482640:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878719.482642:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878719.482643:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878719.482644:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878719.482646:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482648:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878719.482651:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878719.482652:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878719.482654:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878719.482656:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878720.249012:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878720.249019:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878720.249032:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878720.249033:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878720.249120:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878720.249141:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878720.249147:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878721.273076:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878721.273086:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878721.273092:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878721.273097:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878721.273101:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878721.273106:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878721.273110:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878721.273114:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878721.273117:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878721.273119:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878721.273152:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878721.273156:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878721.273159:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878722.297010:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878722.297017:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878722.297024:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878722.297026:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878722.297047:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878722.297050:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878722.297053:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878723.321027:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878723.321037:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878723.321040:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878723.321043:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878723.321049:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878723.321052:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878723.321054:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878723.321056:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878723.321059:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878723.321061:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878723.321088:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878723.321091:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878723.321093:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878724.345054:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878724.345080:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878724.345092:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878724.345096:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878724.345158:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878724.345164:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878724.345169:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878724.601045:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878724.601056:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878724.601059:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878724.601063:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878724.601068:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878724.601076:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9663 00000100:00080000:3.0:1788878724.601088:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878724.601091:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878724.601093:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878724.601101:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878724.601105:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878724.601109:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878724.601112:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878724.601114:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878724.601116:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878724.601120:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878724.601123:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878724.601136:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878724.601140:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878724.601143:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601145:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878724.601148:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878724.601160:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878724.601162:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601165:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878724.601169:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878724.601174:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601186:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878724.601196:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290091904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878724.601207:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878724.601212:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601230:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878724.601233:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878724.601237:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878724.601239:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878724.601241:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878724.601244:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878724.601246:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9663 00000100:00080000:3.0:1788878724.601249:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878724.601251:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878724.601253:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878724.601257:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878724.601259:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878724.601261:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878724.601263:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878724.601266:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878724.601268:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878724.601272:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878724.601274:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878724.601276:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878724.601283:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878724.601285:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601287:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878724.601290:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:3.0:1788878724.601292:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878724.601299:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00000001:3.0:1788878724.601299:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601302:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.601305:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788878724.601308:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878724.601312:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788878724.601312:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.601319:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601319:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878724.601323:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878724.601327:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878724.601329:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878724.601332:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878724.601336:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788878724.601342:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290092032/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788878724.601345:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601360:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1788878724.601361:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290091904:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878724.601365:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601374:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.601376:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788878724.601379:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878724.601395:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:2.0:1788878724.601396:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878724.601397:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878724.601402:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1788878724.601403:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601406:0:177900:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:2.0:1788878724.601411:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788878724.601413:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9663 00000100:00080000:3.0:1788878724.601417:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878724.601419:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878724.601421:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878724.601426:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878724.601428:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878724.601430:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 02000000:00000001:2.0:1788878724.601431:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878724.601433:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878724.601441:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878724.601442:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878724.601445:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878724.601447:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878724.601449:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878724.601452:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878724.601455:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601456:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878724.601459:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000008:00100000:3.0:1788878724.601460:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878724.601466:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000008:00000001:3.0:1788878724.601466:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601472:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.601477:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788878724.601480:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878724.601483:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601485:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878724.601487:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878724.601489:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601495:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878724.601501:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290092160/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878724.601509:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878724.601511:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601514:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878724.601516:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878724.601519:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878724.601521:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878724.601524:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878724.601529:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000800:00000100:2.0:1788878724.601533:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1788878724.601535:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9663 00000100:00080000:3.0:1788878724.601544:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878724.601546:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878724.601548:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878724.601552:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788878724.601554:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:3.0:1788878724.601555:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878724.601559:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878724.601563:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878724.601565:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878724.601566:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878724.601571:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878724.601573:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878724.601576:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878724.601581:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878724.601582:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878724.601587:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878724.601590:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878724.601593:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.601594:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601595:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878724.601599:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878724.601603:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.601606:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304336425 00000100:00000001:3.0:1788878724.601606:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878724.601611:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878724.601614:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601616:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878724.601617:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878724.601619:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.601634:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878724.601635:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878724.601645:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290092288/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878724.601653:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878724.601655:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878724.601657:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.601663:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304336425 00000100:00000001:2.0:1788878724.601675:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878724.601684:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878724.601690:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.601694:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878724.601705:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878724/real 1788878724] req@ffff94e50449e680 x1875765290091904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878724.601726:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878724.601729:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878724.601738:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878724.601743:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878724.601746:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.601751:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878724.601755:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878724.601765:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290091904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878724.601779:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.601806:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878724.601812:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.601815:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878724.601817:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.601821:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878724.601825:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878724.601830:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878724.601846:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878724.601850:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878724.601856:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290091904:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878724.601865:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.601869:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878724.601871:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878724.601875:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878724.601880:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878724.601883:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878724.601886:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878724.601891:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878724.601893:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878724.601897:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878724.601899:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.601939:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.601949:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878724.601952:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.601956:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878724.601958:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878724.601960:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878724.601963:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878724.601965:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878724.601969:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878724.601973:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878724.601977:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878724.601984:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290092032:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878724.601990:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878724.601992:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878724.601995:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878724.601997:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878724.602002:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.602011:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878724.602014:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:2.0:1788878724.602021:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878724.602043:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878724.602050:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878724.602055:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878724.602057:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.602060:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878724.602064:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.602071:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304335884 00000100:00000001:2.0:1788878724.602075:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304336425 00000100:00000001:2.0:1788878724.602079:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304335884 00000100:00000001:2.0:1788878724.602082:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304336425 00000100:00000001:2.0:1788878724.602086:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878724.602089:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.602092:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878724.602100:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878724/real 1788878724] req@ffff94e50449df80 x1875765290092032/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878724.602115:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878724.602117:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878724.602121:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878724.602135:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878724.602138:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602141:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878724.602143:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878724.602150:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290092032/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878724.602162:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602165:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878724.602169:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602172:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878724.602174:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602176:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878724.602195:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878724.602201:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878724.602206:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878724.602210:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878724.602216:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290092032:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878724.602222:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878724.602225:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878724.602227:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878724.602231:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878724.602234:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878724.602237:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878724.602242:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290092160:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878724.602247:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878724.602250:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878724.602252:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878724.602255:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878724.602260:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.602268:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878724.602271:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:2.0:1788878724.602276:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878724.602296:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878724.602303:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878724.602306:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878724.602309:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.602312:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878724.602316:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.602321:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304335882 00000100:00000001:2.0:1788878724.602325:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304336425 00000100:00000001:2.0:1788878724.602328:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304335882 00000100:00000001:2.0:1788878724.602332:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304336425 00000100:00000001:2.0:1788878724.602335:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878724.602338:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.602342:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878724.602349:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878724/real 1788878724] req@ffff94e50449fb80 x1875765290092160/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878724.602363:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878724.602365:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878724.602369:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878724.602372:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878724.602374:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602377:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878724.602380:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878724.602396:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290092160/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878724.602408:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602412:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878724.602415:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602418:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878724.602421:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602423:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878724.602441:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878724.602446:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878724.602452:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878724.602455:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878724.602461:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290092160:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878724.602466:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878724.602468:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878724.602471:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878724.602474:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878724.602477:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878724.602480:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878724.602484:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290092288:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878724.602493:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878724.602496:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878724.602498:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878724.602501:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878724.602505:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.602513:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878724.602516:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:2.0:1788878724.602520:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878724.602539:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878724.602546:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878724.602549:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878724.602552:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.602554:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878724.602558:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.602563:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304335882 00000100:00000001:2.0:1788878724.602568:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304336425 00000100:00000001:2.0:1788878724.602571:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304335882 00000100:00000001:2.0:1788878724.602574:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304336425 00000100:00000001:2.0:1788878724.602578:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878724.602581:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.602583:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878724.602589:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878724/real 1788878724] req@ffff94e50449c380 x1875765290092288/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878724.602602:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878724.602604:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878724.602607:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878724.602610:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878724.602612:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602615:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878724.602617:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878724.602623:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290092288/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878724.602649:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602653:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878724.602657:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602660:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878724.602662:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602665:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878724.602682:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878724.602687:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878724.602692:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878724.602695:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878724.602701:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290092288:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878724.602707:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602711:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878724.602713:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878724.602716:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878724.602719:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878724.602721:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878724.602724:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878724.602727:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878724.602729:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878724.602733:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878724.602735:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602737:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878724.602739:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878724.602741:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878724.602744:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878724.602745:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878724.602747:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878724.602750:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878724.602751:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878724.602754:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878724.602756:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602758:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878724.602760:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878724.602762:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878724.602766:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878724.602768:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878724.602770:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878724.602772:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878724.602774:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878724.602777:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878724.602779:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602782:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878724.602807:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878724.602819:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878724.602822:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878724.602825:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878725.369066:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878725.369076:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878725.369086:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878725.369089:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878725.369132:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878725.369136:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878725.369139:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878726.393054:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878726.393065:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878726.393074:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878726.393077:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878726.393138:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878726.393142:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878726.393146:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878727.417050:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878727.417059:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878727.417066:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878727.417068:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878727.417102:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878727.417106:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878727.417111:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878728.441040:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878728.441046:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878728.441051:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878728.441052:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878728.441132:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878728.441137:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878728.441142:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878729.465062:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878729.465072:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878729.465083:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878729.465085:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878729.465124:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878729.465129:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878729.465134:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878729.721077:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878729.721091:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878729.721095:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878729.721100:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878729.721108:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878729.721129:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9668 00000100:00080000:3.0:1788878729.721144:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878729.721148:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878729.721152:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878729.721164:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878729.721167:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878729.721172:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878729.721176:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878729.721180:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878729.721183:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878729.721192:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878729.721197:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878729.721202:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878729.721207:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878729.721212:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878729.721215:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878729.721219:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878729.721232:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878729.721236:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878729.721240:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878729.721244:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878729.721250:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878729.721265:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878729.721278:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290092416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878729.721293:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878729.721299:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878729.721318:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878729.721322:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878729.721327:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878729.721330:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878729.721333:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878729.721337:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878729.721342:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9668 00000100:00080000:3.0:1788878729.721354:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878729.721357:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878729.721362:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878729.721367:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878729.721371:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878729.721374:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878729.721378:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878729.721383:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878729.721384:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878729.721389:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878729.721393:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788878729.721400:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878729.721406:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878729.721409:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878729.721413:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788878729.721417:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878729.721430:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.721431:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878729.721435:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878729.721438:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788878729.721443:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788878729.721448:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1788878729.721450:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878729.721455:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878729.721456:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878729.721468:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878729.721472:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878729.721478:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878729.721481:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290092416:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878729.721484:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878729.721490:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878729.721494:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878729.721496:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878729.721502:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878729.721506:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878729.721508:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878729.721515:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1788878729.721521:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290092544/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788878729.721525:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878729.721541:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878729.721545:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878729.721550:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.721554:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788878729.721555:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878729.721562:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00080000:3.0:1788878729.721564:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878729.721571:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878729.721572:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788878729.721574:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878729.721580:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000800:00000100:0.0:1788878729.721622:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878729.721636:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878729.721641:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878729.721644:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.721647:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878729.721653:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.721659:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304341545 00000100:00000001:0.0:1788878729.721672:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878729.721680:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304341545 00000100:00000001:0.0:1788878729.721688:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878729.721693:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878729.721696:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.721700:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878729.721709:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878729/real 1788878729] req@ffff94e50449d880 x1875765290092416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878729.721710:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9668 00000100:00080000:3.0:1788878729.721724:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878729.721731:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000400:00000001:0.0:1788878729.721737:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788878729.721738:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:0.0:1788878729.721744:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1788878729.721768:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878729.721773:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878729.721777:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878729.721782:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878729.721783:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878729.721799:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878729.721800:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878729.721804:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878729.721807:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.721815:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1788878729.721819:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878729.721820:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878729.721832:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878729.721839:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:0.0:1788878729.721840:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290092416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788878729.721844:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878729.721854:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878729.721862:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878729.721868:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878729.721870:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1788878729.721878:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.721881:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878729.721884:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.721892:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878729.721896:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878729.721898:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878729.721941:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.721946:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878729.721948:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878729.721954:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878729.721961:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878729.721967:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878729.721973:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290092416:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878729.721982:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.721985:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878729.721987:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878729.721991:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878729.721995:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878729.721998:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878729.722000:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878729.722005:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878729.722007:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878729.722010:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878729.722012:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.722015:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.722024:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878729.722027:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.722029:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878729.722032:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878729.722035:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878729.722048:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878729.722050:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878729.722052:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878729.722055:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878729.722058:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878729.722063:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290092544:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878729.722068:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878729.722071:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878729.722073:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878729.722075:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878729.722079:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.722089:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878729.722092:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:0.0:1788878729.722098:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788878729.722124:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788878729.722128:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878729.722131:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878729.722136:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878729.722140:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.722141:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878729.722149:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878729.722153:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878729.722153:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.722164:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:3.0:1788878729.722172:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290092672/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878729.722173:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.722198:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304341004 00000100:00000001:0.0:1788878729.722204:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304341545 00000100:00000001:0.0:1788878729.722207:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304341004 00000100:00000001:0.0:1788878729.722210:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304341545 00000100:00000001:0.0:1788878729.722213:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878729.722216:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.722220:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878729.722231:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878729/real 1788878729] req@ffff94e50449ce00 x1875765290092544/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878729.722281:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:0.0:1788878729.722282:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788878729.722288:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788878729.722291:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878729.722297:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878729.722304:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878729.722305:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:3.0:1788878729.722313:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878729.722317:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878729.722319:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878729.722322:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878729.722325:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878729.722330:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9668 00000100:00000001:0.0:1788878729.722335:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788878729.722338:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878729.722345:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878729.722346:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878729.722358:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878729.722361:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788878729.722364:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878729.722369:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878729.722372:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:0.0:1788878729.722375:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290092544/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788878729.722377:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878729.722386:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878729.722391:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878729.722404:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878729.722410:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878729.722412:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878729.722431:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878729.722433:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878729.722437:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878729.722439:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878729.722445:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.722446:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878729.722452:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878729.722453:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878729.722460:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000008:00100000:3.0:1788878729.722462:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878729.722470:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878729.722473:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878729.722481:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878729.722484:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878729.722487:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878729.722490:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878729.722491:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878729.722496:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878729.722500:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788878729.722506:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878729.722512:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878729.722521:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788878729.722522:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290092800/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878729.722531:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290092544:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878729.722548:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878729.722557:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878729.722558:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878729.722566:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1788878729.722566:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878729.722573:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788878729.722575:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878729.722579:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878729.722584:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878729.722586:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878729.722591:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878729.722593:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878729.722597:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878729.722600:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.722603:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.722609:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878729.722612:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.722617:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878729.722620:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878729.722622:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878729.722625:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878729.722626:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878729.722629:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878729.722632:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878729.722635:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878729.722639:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290092672:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878729.722643:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878729.722646:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878729.722648:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878729.722650:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878729.722655:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.722663:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878729.722665:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:0.0:1788878729.722671:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878729.722689:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878729.722696:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878729.722699:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878729.722701:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.722703:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878729.722707:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.722711:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304341002 00000100:00000001:0.0:1788878729.722715:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304341545 00000100:00000001:0.0:1788878729.722718:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304341002 00000100:00000001:0.0:1788878729.722721:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304341545 00000100:00000001:0.0:1788878729.722723:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878729.722726:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.722728:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878729.722733:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878729/real 1788878729] req@ffff94e50449d180 x1875765290092672/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878729.722743:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878729.722744:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878729.722776:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878729.722782:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878729.722785:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.722787:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878729.722789:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878729.722795:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290092672/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878729.722804:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.722807:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878729.722810:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.722812:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878729.722814:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.722816:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878729.722829:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878729.722833:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878729.722838:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878729.722841:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878729.722845:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290092672:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878729.722849:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878729.722851:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878729.722852:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878729.722855:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878729.722857:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878729.722864:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878729.722868:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290092800:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878729.722872:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878729.722875:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878729.722877:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878729.722879:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878729.722883:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.722890:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878729.722892:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:0.0:1788878729.722896:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878729.722944:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878729.722954:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878729.722959:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878729.722962:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.722964:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878729.722968:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.722974:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304341003 00000100:00000001:0.0:1788878729.722978:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304341545 00000100:00000001:0.0:1788878729.722981:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304341003 00000100:00000001:0.0:1788878729.722985:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304341545 00000100:00000001:0.0:1788878729.722988:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878729.722991:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.722995:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878729.723004:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878729/real 1788878729] req@ffff94e50449f480 x1875765290092800/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878729.723015:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878729.723018:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878729.723021:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878729.723024:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878729.723026:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.723035:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878729.723037:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878729.723042:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290092800/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878729.723050:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.723053:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878729.723056:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.723058:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878729.723059:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.723061:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878729.723073:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878729.723076:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878729.723080:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878729.723082:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878729.723086:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290092800:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878729.723090:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.723092:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878729.723094:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878729.723096:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878729.723104:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878729.723106:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878729.723108:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878729.723111:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878729.723112:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878729.723115:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878729.723117:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.723119:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878729.723121:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878729.723123:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878729.723126:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878729.723127:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878729.723128:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878729.723130:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878729.723131:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878729.723134:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878729.723135:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.723137:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878729.723141:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878729.723143:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878729.723145:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878729.723147:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878730.488986:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878730.488992:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878730.488998:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878730.488999:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878730.489035:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878730.489038:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878730.489041:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878731.512994:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878731.513001:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878731.513007:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878731.513011:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878731.513014:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878731.513018:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878731.513021:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878731.513024:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878731.513026:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878731.513029:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878731.513051:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878731.513054:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878731.513057:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878732.536998:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878732.537005:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878732.537010:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878732.537012:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878732.537071:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878732.537075:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878732.537079:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878733.561037:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878733.561064:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878733.561066:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878733.561069:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878733.561074:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878733.561077:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878733.561079:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878733.561081:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878733.561084:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878733.561085:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878733.561107:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878733.561109:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878733.561112:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878734.584979:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878734.584994:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878734.584998:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878734.585000:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878734.585017:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878734.585019:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878734.585021:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878734.841015:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878734.841043:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878734.841046:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878734.841050:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878734.841056:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878734.841063:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9673 00000100:00080000:3.0:1788878734.841076:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878734.841080:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878734.841083:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878734.841091:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878734.841093:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878734.841097:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878734.841099:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878734.841102:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878734.841103:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878734.841107:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878734.841110:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878734.841113:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878734.841116:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878734.841118:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878734.841120:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878734.841122:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878734.841133:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878734.841134:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878734.841153:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878734.841155:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878734.841159:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878734.841170:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878734.841178:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290092928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878734.841187:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878734.841198:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878734.841212:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878734.841231:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878734.841234:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878734.841235:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878734.841236:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878734.841239:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878734.841241:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9673 00000100:00080000:3.0:1788878734.841244:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878734.841246:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878734.841247:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878734.841250:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878734.841252:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878734.841254:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878734.841257:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878734.841259:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878734.841260:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878734.841263:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878734.841266:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878734.841267:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878734.841273:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878734.841275:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841276:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878734.841277:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878734.841283:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878734.841284:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878734.841289:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00000001:3.0:1788878734.841289:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841295:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878734.841295:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841300:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878734.841303:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878734.841305:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878734.841306:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878734.841310:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878734.841311:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878734.841312:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878734.841316:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878734.841317:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878734.841323:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878734.841325:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290092928:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1788878734.841332:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290093056/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878734.841333:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878734.841339:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878734.841343:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878734.841343:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878734.841348:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878734.841349:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878734.841353:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878734.841356:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878734.841356:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878734.841361:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878734.841364:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878734.841365:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878734.841367:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878734.841369:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9673 00000100:00080000:3.0:1788878734.841372:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878734.841374:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878734.841376:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878734.841376:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878734.841379:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878734.841380:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000020:00000001:3.0:1788878734.841382:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878734.841385:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878734.841387:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1788878734.841389:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878734.841392:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878734.841393:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878734.841396:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878734.841398:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878734.841399:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878734.841402:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878734.841404:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878734.841405:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878734.841407:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878734.841409:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878734.841410:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878734.841414:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878734.841415:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878734.841417:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878734.841417:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878734.841419:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878734.841423:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788878734.841428:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1788878734.841432:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290093184/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878734.841440:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878734.841442:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841443:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878734.841445:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878734.841448:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878734.841449:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1788878734.841452:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878734.841454:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878734.841455:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878734.841458:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878734.841460:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878734.841461:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878734.841466:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9673 00000100:00000001:0.0:1788878734.841467:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878734.841469:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878734.841473:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878734.841474:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304346665 00000100:00000001:3.0:1788878734.841475:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878734.841481:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878734.841483:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878734.841484:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878734.841486:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878734.841488:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878734.841489:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878734.841490:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878734.841511:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878734.841513:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878734.841515:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878734.841517:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304346665 02000000:00000001:3.0:1788878734.841518:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878734.841522:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878734.841524:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878734.841525:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000008:00100000:3.0:1788878734.841526:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878734.841531:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841532:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878734.841534:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841538:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878734.841540:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878734.841542:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1788878734.841543:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878734.841546:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878734.841548:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1788878734.841552:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878734/real 1788878734] req@ffff94e50449c700 x1875765290092928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788878734.841553:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878734.841558:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878734.841584:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290093312/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1788878734.841588:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878734.841591:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878734.841596:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878734.841597:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878734.841601:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841602:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878734.841605:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841606:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.841610:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878734.841612:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878734.841618:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290092928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878734.841624:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.841627:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878734.841630:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.841631:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878734.841632:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.841635:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878734.841637:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878734.841640:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878734.841644:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878734.841647:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878734.841650:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290092928:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878734.841654:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.841656:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878734.841657:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878734.841659:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878734.841662:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878734.841663:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878734.841665:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878734.841668:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878734.841670:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878734.841672:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878734.841673:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.841675:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.841680:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878734.841682:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841683:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878734.841685:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878734.841686:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878734.841687:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878734.841688:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878734.841690:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878734.841691:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878734.841693:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878734.841695:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290093056:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878734.841698:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878734.841700:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878734.841701:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878734.841703:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878734.841705:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841709:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878734.841711:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:0.0:1788878734.841733:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878734.841748:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878734.841754:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878734.841757:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878734.841759:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841761:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878734.841764:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841769:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304346123 00000100:00000001:0.0:1788878734.841772:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304346665 00000100:00000001:0.0:1788878734.841774:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304346123 00000100:00000001:0.0:1788878734.841776:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304346665 00000100:00000001:0.0:1788878734.841777:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878734.841779:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841781:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878734.841784:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878734/real 1788878734] req@ffff94e50449f800 x1875765290093056/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878734.841790:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878734.841792:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878734.841794:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878734.841795:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878734.841796:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.841798:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878734.841799:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878734.841802:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290093056/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878734.841807:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.841808:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878734.841810:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.841811:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878734.841812:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.841813:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878734.841823:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878734.841825:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878734.841827:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878734.841830:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878734.841832:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290093056:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878734.841835:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878734.841836:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878734.841837:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878734.841839:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878734.841840:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878734.841842:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878734.841844:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290093184:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878734.841846:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878734.841848:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878734.841849:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878734.841850:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878734.841853:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841857:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878734.841858:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:0.0:1788878734.841861:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878734.841870:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878734.841873:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878734.841875:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878734.841876:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841894:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878734.841896:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841899:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304346121 00000100:00000001:0.0:1788878734.841901:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304346665 00000100:00000001:0.0:1788878734.841903:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304346121 00000100:00000001:0.0:1788878734.841904:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304346665 00000100:00000001:0.0:1788878734.841906:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878734.841907:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.841909:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878734.841912:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878734/real 1788878734] req@ffff94e50449c000 x1875765290093184/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878734.841957:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878734.841958:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878734.841960:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878734.841962:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878734.841963:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.841964:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878734.841965:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878734.841969:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290093184/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878734.841991:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.841993:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878734.841995:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.841996:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878734.841997:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.841998:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878734.842007:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878734.842009:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878734.842011:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878734.842012:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878734.842015:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290093184:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878734.842018:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878734.842019:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878734.842020:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878734.842022:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878734.842023:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878734.842025:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878734.842027:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290093312:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878734.842029:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878734.842031:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878734.842032:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878734.842034:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878734.842036:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.842041:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878734.842042:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:0.0:1788878734.842045:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878734.842055:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878734.842059:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878734.842060:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878734.842061:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.842063:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878734.842065:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.842067:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304346122 00000100:00000001:0.0:1788878734.842069:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304346665 00000100:00000001:0.0:1788878734.842071:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304346122 00000100:00000001:0.0:1788878734.842073:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304346665 00000100:00000001:0.0:1788878734.842075:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878734.842076:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.842078:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878734.842080:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878734/real 1788878734] req@ffff94e50449ca80 x1875765290093312/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878734.842086:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878734.842088:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878734.842090:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878734.842092:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878734.842094:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.842096:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878734.842098:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878734.842102:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290093312/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878734.842111:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.842113:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878734.842116:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.842119:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878734.842120:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.842123:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878734.842137:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878734.842141:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878734.842161:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878734.842164:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878734.842167:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290093312:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878734.842170:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.842172:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878734.842173:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878734.842174:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878734.842176:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878734.842178:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878734.842179:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878734.842180:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878734.842182:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878734.842183:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878734.842184:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.842186:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878734.842187:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878734.842187:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878734.842189:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878734.842189:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878734.842190:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878734.842191:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878734.842192:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878734.842193:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878734.842194:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.842195:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878734.842196:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878734.842197:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878734.842198:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878734.842199:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878734.842199:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878734.842200:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878734.842201:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878734.842202:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878734.842204:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.842205:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878734.842208:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878734.842209:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878734.842211:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878734.842228:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878735.608988:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878735.608993:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878735.608997:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878735.608998:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878735.609014:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878735.609016:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878735.609018:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878736.633046:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878736.633054:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878736.633059:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878736.633060:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878736.633202:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878736.633211:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878736.633219:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878737.657062:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878737.657070:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878737.657076:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878737.657078:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878737.657117:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878737.657120:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878737.657124:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878738.681000:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878738.681008:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878738.681014:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878738.681015:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878738.681037:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878738.681040:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878738.681042:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878739.705060:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878739.705070:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878739.705078:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878739.705080:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878739.705111:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878739.705116:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878739.705130:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878739.961117:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878739.961130:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878739.961133:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878739.961137:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878739.961142:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878739.961152:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9678 00000100:00080000:0.0:1788878739.961164:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878739.961168:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878739.961169:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878739.961180:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878739.961183:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878739.961187:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878739.961189:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878739.961191:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878739.961192:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878739.961196:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878739.961198:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878739.961201:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878739.961204:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878739.961207:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961209:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878739.961211:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878739.961234:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878739.961236:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961239:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878739.961242:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878739.961248:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961262:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878739.961272:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290093440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878739.961288:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878739.961293:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961313:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878739.961315:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878739.961318:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878739.961319:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878739.961321:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878739.961323:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878739.961325:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9678 00000100:00080000:0.0:1788878739.961328:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878739.961330:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878739.961331:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878739.961333:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878739.961334:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878739.961336:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788878739.961338:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878739.961340:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878739.961341:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878739.961343:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878739.961345:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878739.961346:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878739.961352:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878739.961353:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961355:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878739.961359:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878739.961361:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961363:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878739.961366:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878739.961367:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961368:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878739.961369:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878739.961371:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961375:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878739.961379:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290093568/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878739.961386:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878739.961388:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961390:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878739.961392:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878739.961395:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878739.961397:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878739.961398:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878739.961399:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878739.961401:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878739.961406:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9678 00000100:00000001:2.0:1788878739.961407:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788878739.961408:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878739.961411:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878739.961412:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878739.961413:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878739.961417:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788878739.961417:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878739.961421:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878739.961422:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878739.961425:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878739.961428:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 02000000:00000001:2.0:1788878739.961429:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788878739.961433:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1788878739.961434:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878739.961435:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788878739.961439:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878739.961441:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878739.961444:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878739.961446:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1788878739.961447:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290093440:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788878739.961449:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878739.961455:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.961457:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788878739.961458:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878739.961462:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878739.961465:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878739.961465:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788878739.961469:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878739.961470:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878739.961472:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878739.961474:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878739.961475:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961478:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878739.961478:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878739.961480:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878739.961481:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961485:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878739.961490:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979180 x1875765290093696/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878739.961495:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878739.961497:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961499:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878739.961501:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788878739.961503:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1788878739.961504:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878739.961508:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 00000100:00000001:0.0:1788878739.961509:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878739.961512:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878739.961516:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788878739.961516:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788878739.961521:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9678 00000100:00080000:0.0:1788878739.961523:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878739.961525:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878739.961526:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878739.961528:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878739.961529:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878739.961530:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788878739.961532:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878739.961533:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878739.961534:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878739.961537:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878739.961557:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878739.961559:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878739.961562:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878739.961563:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961566:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878739.961568:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878739.961570:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961572:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878739.961580:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878739.961581:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961583:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878739.961584:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878739.961586:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961593:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788878739.961595:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1788878739.961602:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290093824/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878739.961611:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878739.961614:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878739.961617:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.961617:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878739.961626:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878739.961630:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.961633:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878739.961640:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.961646:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304351785 00000100:00000001:2.0:1788878739.961661:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878739.961707:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304351785 00000100:00000001:2.0:1788878739.961715:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878739.961721:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878739.961724:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.961728:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878739.961735:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878739/real 1788878739] req@ffff94e508978380 x1875765290093440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878739.961750:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878739.961752:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878739.961758:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878739.961761:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878739.961763:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.961766:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878739.961768:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878739.961773:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290093440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878739.961782:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.961786:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878739.961789:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.961791:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878739.961793:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.961795:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878739.961798:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878739.961801:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878739.961805:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878739.961808:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878739.961812:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290093440:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878739.961817:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878739.961818:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878739.961819:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878739.961822:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878739.961824:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878739.961827:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878739.961830:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290093568:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878739.961834:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878739.961836:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878739.961837:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878739.961839:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878739.961843:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.961849:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878739.961851:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000100:00000001:2.0:1788878739.961856:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878739.961869:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878739.961873:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878739.961875:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878739.961877:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.961879:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878739.961882:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.961886:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304351243 00000100:00000001:2.0:1788878739.961890:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304351785 00000100:00000001:2.0:1788878739.961892:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304351243 00000100:00000001:2.0:1788878739.961895:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304351785 00000100:00000001:2.0:1788878739.961897:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878739.961899:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.961900:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878739.961950:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878739/real 1788878739] req@ffff94e50897b800 x1875765290093568/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878739.961959:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878739.961960:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878739.961963:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878739.961965:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878739.961966:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.961968:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878739.961970:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878739.961975:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290093568/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878739.961982:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.961984:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878739.961987:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.961988:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878739.961990:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.961991:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878739.962011:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878739.962030:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878739.962034:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878739.962037:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878739.962041:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290093568:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878739.962047:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.962050:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878739.962052:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878739.962056:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878739.962061:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878739.962064:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878739.962067:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878739.962072:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878739.962074:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878739.962078:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878739.962080:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.962082:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878739.962084:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878739.962085:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878739.962088:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878739.962089:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878739.962091:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878739.962093:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878739.962094:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878739.962098:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878739.962099:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.962102:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.962113:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878739.962116:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.962119:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878739.962121:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878739.962123:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878739.962125:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878739.962127:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878739.962130:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878739.962132:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878739.962135:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878739.962139:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290093696:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878739.962144:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878739.962146:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878739.962148:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878739.962151:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878739.962155:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.962163:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878739.962165:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 00000100:00000001:2.0:1788878739.962171:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878739.962187:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878739.962192:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878739.962195:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878739.962197:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.962199:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878739.962202:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.962207:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304351242 00000100:00000001:2.0:1788878739.962210:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304351785 00000100:00000001:2.0:1788878739.962214:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304351242 00000100:00000001:2.0:1788878739.962217:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304351785 00000100:00000001:2.0:1788878739.962220:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878739.962222:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.962224:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878739.962229:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878739/real 1788878739] req@ffff94e508979180 x1875765290093696/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878739.962248:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878739.962249:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878739.962252:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878739.962254:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878739.962256:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.962258:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878739.962259:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878739.962264:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290093696/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878739.962271:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.962274:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878739.962277:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.962279:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878739.962280:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.962283:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878739.962300:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878739.962303:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878739.962307:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878739.962326:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878739.962330:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290093696:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878739.962333:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878739.962334:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878739.962336:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878739.962338:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878739.962339:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878739.962341:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878739.962344:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290093824:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878739.962347:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878739.962349:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878739.962351:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878739.962353:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878739.962356:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.962360:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878739.962362:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:2.0:1788878739.962366:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878739.962376:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878739.962380:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878739.962382:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878739.962384:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.962385:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878739.962388:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.962391:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304351242 00000100:00000001:2.0:1788878739.962394:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304351785 00000100:00000001:2.0:1788878739.962396:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304351242 00000100:00000001:2.0:1788878739.962398:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304351785 00000100:00000001:2.0:1788878739.962401:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878739.962403:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.962404:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878739.962408:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878739/real 1788878739] req@ffff94e508979c00 x1875765290093824/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878739.962433:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878739.962434:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878739.962437:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878739.962438:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878739.962440:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.962442:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878739.962444:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878739.962448:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290093824/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878739.962456:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.962459:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878739.962462:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.962464:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878739.962466:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.962468:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878739.962486:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878739.962490:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878739.962494:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878739.962497:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878739.962500:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290093824:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878739.962504:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.962506:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878739.962508:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878739.962510:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878739.962512:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878739.962513:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878739.962515:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878739.962518:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878739.962519:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878739.962540:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878739.962541:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.962543:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878739.962545:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878739.962546:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878739.962548:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878739.962550:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878739.962551:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878739.962553:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878739.962555:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878739.962556:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878739.962558:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.962560:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878739.962563:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878739.962565:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878739.962567:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878739.962569:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878740.729056:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878740.729066:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878740.729073:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878740.729075:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878740.729153:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878740.729159:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878740.729165:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878741.753131:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878741.753144:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878741.753171:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878741.753177:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878741.753181:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878741.753188:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878741.753194:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878741.753198:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878741.753201:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878741.753206:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878741.753243:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878741.753248:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878741.753253:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878742.777064:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878742.777075:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878742.777083:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878742.777085:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878742.777116:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878742.777120:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878742.777124:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878743.801035:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878743.801045:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878743.801047:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878743.801050:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878743.801055:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878743.801057:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878743.801059:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878743.801061:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878743.801065:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878743.801066:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878743.801088:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878743.801091:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878743.801093:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878744.824977:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878744.824985:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878744.824991:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878744.824992:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878744.825059:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878744.825065:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878744.825069:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878745.081022:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878745.081034:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878745.081037:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878745.081041:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878745.081046:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878745.081054:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9683 00000100:00080000:2.0:1788878745.081067:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878745.081069:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878745.081071:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878745.081081:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878745.081083:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878745.081087:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878745.081089:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878745.081101:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878745.081102:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878745.081106:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878745.081107:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878745.081110:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878745.081114:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878745.081116:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081118:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878745.081120:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878745.081132:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878745.081134:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081137:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878745.081140:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878745.081145:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081160:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878745.081170:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290093952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878745.081180:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878745.081185:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081202:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878745.081206:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878745.081209:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878745.081210:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878745.081211:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878745.081214:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878745.081216:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9683 00000100:00080000:2.0:1788878745.081218:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878745.081220:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878745.081221:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878745.081224:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878745.081225:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878745.081227:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878745.081229:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878745.081231:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878745.081232:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878745.081234:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878745.081236:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878745.081237:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878745.081239:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878745.081244:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788878745.081246:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878745.081249:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.081250:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878745.081251:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878745.081256:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:2.0:1788878745.081258:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878745.081261:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788878745.081262:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081267:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878745.081269:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878745.081270:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878745.081272:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878745.081273:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878745.081274:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878745.081277:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878745.081278:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878745.081280:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081285:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878745.081286:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290093952:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1788878745.081293:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290094080/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878745.081294:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878745.081300:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878745.081303:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878745.081308:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788878745.081311:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878745.081313:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878745.081315:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081316:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878745.081320:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878745.081323:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878745.081323:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878745.081333:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878745.081335:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878745.081338:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9683 00000100:00080000:2.0:1788878745.081341:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878745.081342:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878745.081343:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878745.081345:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878745.081346:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878745.081348:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878745.081349:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1788878745.081350:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878745.081352:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:2.0:1788878745.081354:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878745.081357:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878745.081358:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788878745.081363:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878745.081365:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878745.081366:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878745.081368:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878745.081369:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081370:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878745.081372:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878745.081373:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081374:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878745.081378:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878745.081379:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081381:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878745.081381:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878745.081383:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081386:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878745.081389:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290094208/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878745.081395:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878745.081396:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081398:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878745.081399:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878745.081402:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878745.081403:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878745.081404:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1788878745.081408:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878745.081409:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878745.081439:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9683 00000100:00080000:2.0:1788878745.081445:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878745.081447:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878745.081449:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878745.081452:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:2.0:1788878745.081453:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878745.081458:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878745.081462:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1788878745.081462:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878745.081466:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878745.081468:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081471:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878745.081473:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878745.081475:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878745.081482:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878745.081484:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081486:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878745.081490:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878745.081492:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304356905 02000000:00000001:2.0:1788878745.081493:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878745.081502:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081504:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878745.081521:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878745.081523:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081524:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.081528:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1788878745.081528:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878745.081529:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081533:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878745.081534:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878745.081535:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.081539:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304356905 00000100:00000001:2.0:1788878745.081540:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878745.081546:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290094336/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878745.081548:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878745.081553:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878745.081554:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878745.081558:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878745.081559:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.081564:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878745.081564:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878745.081572:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878745/real 1788878745] req@ffff94e5045e4700 x1875765290093952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878745.081583:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878745.081585:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878745.081590:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878745.081592:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878745.081594:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.081596:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878745.081597:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878745.081601:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290093952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878745.081606:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.081609:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878745.081612:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.081613:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878745.081614:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.081616:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878745.081618:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878745.081621:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878745.081625:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878745.081627:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878745.081630:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290093952:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878745.081634:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.081636:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878745.081637:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878745.081639:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878745.081659:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878745.081662:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878745.081665:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878745.081669:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878745.081672:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878745.081686:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878745.081687:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.081691:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.081699:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878745.081701:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.081704:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878745.081706:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878745.081708:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878745.081710:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878745.081711:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878745.081714:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878745.081716:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878745.081718:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878745.081722:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290094080:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878745.081725:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878745.081726:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878745.081727:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878745.081729:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878745.081731:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.081737:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878745.081738:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00000001:0.0:1788878745.081743:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878745.081753:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878745.081756:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878745.081758:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878745.081759:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.081760:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878745.081762:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.081766:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304356363 00000100:00000001:0.0:1788878745.081769:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304356905 00000100:00000001:0.0:1788878745.081770:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304356363 00000100:00000001:0.0:1788878745.081772:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304356905 00000100:00000001:0.0:1788878745.081774:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878745.081775:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.081776:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878745.081779:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878745/real 1788878745] req@ffff94e5045e7480 x1875765290094080/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878745.081784:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878745.081785:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878745.081786:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878745.081787:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878745.081789:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.081790:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878745.081791:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878745.081794:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290094080/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878745.081799:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.081800:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878745.081802:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.081803:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878745.081803:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.081805:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878745.081821:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878745.081823:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878745.081826:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878745.081827:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878745.081830:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290094080:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878745.081832:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878745.081833:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878745.081833:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878745.081835:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878745.081836:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878745.081837:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878745.081839:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290094208:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878745.081842:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878745.081843:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878745.081844:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878745.081845:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878745.081847:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.081850:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878745.081851:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00000001:0.0:1788878745.081853:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878745.081859:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878745.081861:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878745.081863:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878745.081864:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.081865:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878745.081867:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.081869:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304356361 00000100:00000001:0.0:1788878745.081871:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304356905 00000100:00000001:0.0:1788878745.081872:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304356361 00000100:00000001:0.0:1788878745.081874:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304356905 00000100:00000001:0.0:1788878745.081876:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878745.081877:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.081878:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878745.081881:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878745/real 1788878745] req@ffff94e5045e7b80 x1875765290094208/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878745.081886:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878745.081887:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878745.081889:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878745.081890:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878745.081891:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.081892:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878745.081893:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878745.081896:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290094208/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878745.081900:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.081918:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878745.081920:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.081921:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878745.081921:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.081922:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878745.081936:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878745.081938:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878745.081939:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878745.081941:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878745.081943:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290094208:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878745.081946:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878745.081947:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878745.081948:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878745.081949:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878745.081951:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878745.081952:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878745.081954:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290094336:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878745.081958:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878745.081959:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878745.081961:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878745.081963:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878745.081966:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.081971:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878745.081973:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:0.0:1788878745.081977:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878745.081989:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878745.081994:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878745.081996:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878745.081998:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.082000:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878745.082002:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.082006:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304356362 00000100:00000001:0.0:1788878745.082009:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304356905 00000100:00000001:0.0:1788878745.082011:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304356362 00000100:00000001:0.0:1788878745.082014:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304356905 00000100:00000001:0.0:1788878745.082016:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878745.082018:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.082019:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878745.082022:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878745/real 1788878745] req@ffff94e5045e4e00 x1875765290094336/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878745.082028:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878745.082029:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878745.082031:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878745.082033:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878745.082034:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.082035:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878745.082036:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878745.082039:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290094336/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878745.082048:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.082050:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878745.082052:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.082053:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878745.082053:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.082054:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878745.082068:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878745.082069:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878745.082071:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878745.082072:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878745.082074:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290094336:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878745.082077:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.082079:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878745.082080:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878745.082081:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878745.082083:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878745.082084:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878745.082085:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878745.082086:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878745.082087:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878745.082089:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878745.082090:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.082091:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878745.082092:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878745.082093:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878745.082105:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878745.082106:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878745.082107:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878745.082109:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878745.082110:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878745.082111:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878745.082111:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.082112:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878745.082113:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878745.082114:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878745.082115:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878745.082116:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878745.082117:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878745.082118:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878745.082119:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878745.082120:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878745.082121:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.082122:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878745.082125:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878745.082126:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878745.082127:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878745.082128:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878745.848971:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878745.848977:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878745.848981:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878745.848982:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878745.848999:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878745.849001:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878745.849003:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878746.873016:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878746.873021:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878746.873025:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878746.873026:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878746.873045:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878746.873047:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878746.873049:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878747.897028:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878747.897036:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878747.897042:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878747.897044:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878747.897078:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878747.897082:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878747.897087:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878748.921051:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878748.921059:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878748.921065:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878748.921067:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878748.921213:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878748.921222:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878748.921229:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878749.944981:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878749.944987:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878749.944991:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878749.944992:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878749.945007:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878749.945009:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878749.945011:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878750.201014:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878750.201029:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878750.201030:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878750.201034:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878750.201053:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878750.201059:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9689 00000100:00080000:3.0:1788878750.201068:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878750.201071:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878750.201073:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878750.201079:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878750.201082:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878750.201085:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878750.201087:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878750.201089:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878750.201090:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878750.201093:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878750.201095:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878750.201097:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878750.201100:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878750.201102:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201104:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878750.201106:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878750.201116:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878750.201117:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201119:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878750.201122:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878750.201125:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201136:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878750.201143:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290094464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878750.201152:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878750.201155:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201170:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878750.201173:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878750.201177:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878750.201179:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878750.201181:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878750.201184:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878750.201194:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9689 00000100:00080000:3.0:1788878750.201199:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878750.201203:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878750.201205:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878750.201210:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878750.201211:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788878750.201212:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878750.201217:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788878750.201217:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878750.201221:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788878750.201223:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878750.201224:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878750.201227:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201230:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1788878750.201232:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878750.201234:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:0.0:1788878750.201235:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878750.201240:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878750.201241:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1788878750.201247:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290094464:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878750.201249:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201254:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878750.201255:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878750.201258:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878750.201261:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878750.201264:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878750.201264:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878750.201271:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878750.201271:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201278:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878750.201283:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878750.201288:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201306:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878750.201307:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201309:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 02000000:00000001:3.0:1788878750.201313:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878750.201314:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788878750.201317:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201319:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878750.201320:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878750.201322:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201328:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878750.201333:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290094592/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878750.201339:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878750.201340:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201343:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878750.201345:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878750.201348:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878750.201349:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878750.201350:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878750.201352:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878750.201354:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9689 00000100:00000001:0.0:1788878750.201356:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878750.201358:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878750.201361:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304362025 00000100:00000001:3.0:1788878750.201361:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878750.201364:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878750.201369:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878750.201370:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878750.201372:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878750.201373:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1788878750.201375:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878750.201379:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878750.201380:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878750.201381:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304362025 00000100:00080000:3.0:1788878750.201384:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878750.201387:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878750.201388:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878750.201391:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878750.201393:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878750.201395:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878750.201396:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878750.201398:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201401:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201401:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878750.201404:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878750.201405:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1788878750.201410:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201413:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201414:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878750.201417:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:3.0:1788878750.201418:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878750.201419:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788878750.201421:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201422:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878750.201424:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878750.201427:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878750.201428:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878750.201430:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878750.201432:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878750.201433:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878750.201437:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878750.201438:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878750.201439:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878750.201445:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290094592:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878750.201448:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788878750.201448:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878750.201451:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878750.201453:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878750.201455:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878750.201459:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878750.201459:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290094720/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878750.201467:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878750.201468:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:0.0:1788878750.201472:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878750.201478:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878750.201480:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201482:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201484:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201487:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304361483 00000100:00080000:3.0:1788878750.201488:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878750.201492:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304362025 00000100:00080000:3.0:1788878750.201493:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878750.201496:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304361483 00000100:00000001:3.0:1788878750.201496:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878750.201500:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304362025 00000100:00080000:3.0:1788878750.201500:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878750.201506:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878750.201507:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878750.201510:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878750.201513:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9689 00000100:00000001:0.0:1788878750.201514:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201517:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788878750.201519:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878750.201520:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878750.201522:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788878750.201523:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878750.201525:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878750.201526:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878750.201527:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878750.201531:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788878750.201532:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878750.201533:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1788878750.201535:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878750.201536:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878750.201538:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:0.0:1788878750.201539:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878750.201542:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788878750.201543:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:0.0:1788878750.201545:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201547:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878750.201550:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:0.0:1788878750.201551:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290094720:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878750.201556:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788878750.201557:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:0.0:1788878750.201558:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878750.201561:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201561:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878750.201566:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788878750.201566:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878750.201570:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878750.201571:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878750.201574:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788878750.201576:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201577:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:0.0:1788878750.201582:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878750.201583:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878750.201587:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878750.201590:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788878750.201591:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201594:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304361481 00000100:00000001:3.0:1788878750.201595:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201597:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304362025 00000100:00000001:0.0:1788878750.201600:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304361481 00000100:00000001:0.0:1788878750.201601:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304362025 02000000:00000001:3.0:1788878750.201603:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878750.201604:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878750.201620:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201623:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878750.201625:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878750.201627:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201633:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878750.201637:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290094848/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878750.201642:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878750.201644:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878750.201646:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201653:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201656:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201657:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878750.201659:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878750.201661:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788878750.201662:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878750.201664:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878750.201665:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878750.201666:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878750.201667:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878750.201669:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878750.201670:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878750.201672:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878750.201675:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290094848:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878750.201677:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878750.201679:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878750.201680:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878750.201681:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878750.201684:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201688:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878750.201689:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:0.0:1788878750.201691:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788878750.201699:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201701:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304361481 00000100:00000001:0.0:1788878750.201703:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304362025 00000100:00000001:0.0:1788878750.201705:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304361481 00000100:00000001:0.0:1788878750.201708:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304362025 00000100:00000001:0.0:1788878750.201710:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878750.201711:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201714:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201715:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878750.201718:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878750.201719:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788878750.201721:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878750.201722:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878750.201724:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878750.201726:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878750.968954:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878750.968971:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878750.968975:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878750.968976:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878750.968991:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878750.968992:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878750.968995:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878751.992979:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878751.992984:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878751.992988:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878751.992990:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878751.992999:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878751.993013:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878751.993015:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878751.993017:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878751.993018:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878751.993020:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878751.993036:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878751.993038:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878751.993040:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878753.017005:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878753.017011:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878753.017015:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878753.017017:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878753.017127:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878753.017131:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878753.017134:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788878753.273242:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:2.0:1788878753.273304:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788878753.273322:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788878753.273338:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878753.273363:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878753.273366:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878753.273369:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:2.0:1788878753.273379:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788878753.273384:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878753.273387:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878753.273388:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878753.273391:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:2.0:1788878753.273396:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788878753.273401:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878753.273406:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878753.273407:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878753.273410:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:2.0:1788878753.273420:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788878753.273430:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878753.273431:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878753.273437:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878753.273441:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878753.273443:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878753.273450:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878753.273459:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:0.0:1788878753.273477:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878750/real 1788878753] req@ffff94e50449c000 x1875765290094848/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878753.273495:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878753.273498:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878753.273506:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878753.273509:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878753.273511:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.273514:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878753.273517:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878753.273525:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290094848/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878753.273539:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.273545:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878753.273550:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.273553:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878753.273555:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.273559:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878753.273580:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878753.273610:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878753.273619:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878753.273624:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878753.273632:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290094848:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878753.273645:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878753.273652:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878750/real 1788878753] req@ffff94e50449ca80 x1875765290094720/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878753.273668:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878753.273670:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878753.273677:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878753.273680:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878753.273682:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.273685:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878753.273687:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878753.273695:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290094720/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878753.273709:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.273712:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878753.273717:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.273720:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878753.273722:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.273725:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878753.273745:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878753.273750:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878753.273756:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878753.273760:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878753.273773:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290094720:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878753.273785:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878753.273793:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878750/real 1788878753] req@ffff94e50449f100 x1875765290094592/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878753.273807:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878753.273809:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878753.273815:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878753.273819:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878753.273821:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.273824:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878753.273826:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878753.273832:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290094592/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878753.273844:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.273847:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878753.273852:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.273854:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878753.273856:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.273859:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878753.273938:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878753.273944:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878753.273951:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878753.273955:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878753.273961:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290094592:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878753.273968:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878753.273976:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878750/real 1788878753] req@ffff94e50449e300 x1875765290094464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878753.273991:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878753.273993:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878753.273998:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878753.274002:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878753.274004:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.274007:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878753.274010:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878753.274017:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290094464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878753.274029:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.274033:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878753.274037:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.274050:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878753.274052:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.274056:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878753.274058:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878753.274063:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878753.274069:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878753.274073:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878753.274078:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290094464:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878753.274084:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.274088:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878753.274091:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878753.274095:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878753.274103:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878753.274108:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878753.274112:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878753.274119:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878753.274123:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878753.274128:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878753.274132:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.274136:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878753.274139:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878753.274141:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878753.274146:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878753.274148:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878753.274151:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878753.274155:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878753.274157:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878753.274162:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878753.274165:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.274169:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878753.274172:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878753.274175:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878753.274179:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878753.274182:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878753.274184:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878753.274188:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878753.274190:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878753.274194:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878753.274196:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.274199:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878753.274201:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878753.274203:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878753.274205:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878753.274207:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878753.274209:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878753.274213:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878753.274215:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878753.274217:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878753.274220:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.274224:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878753.274235:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878753.274238:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878753.274243:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878753.274246:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878754.041013:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878754.041020:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878754.041022:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878754.041024:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878754.041028:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878754.041030:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878754.041032:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878754.041033:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878754.041036:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878754.041037:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878754.041053:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878754.041055:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878754.041057:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878755.065094:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878755.065116:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878755.065123:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878755.065124:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878755.065171:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878755.065176:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878755.065183:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878755.321059:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878755.321074:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878755.321077:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878755.321081:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878755.321088:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878755.321098:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9694 00000100:00080000:0.0:1788878755.321112:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878755.321116:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878755.321119:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878755.321130:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878755.321133:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878755.321138:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878755.321140:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878755.321143:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878755.321157:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878755.321167:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878755.321171:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878755.321174:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878755.321179:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878755.321183:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878755.321186:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878755.321189:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878755.321202:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878755.321203:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878755.321208:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878755.321211:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878755.321218:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878755.321235:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878755.321247:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290094976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878755.321261:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878755.321267:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878755.321302:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878755.321307:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878755.321314:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878755.321316:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878755.321320:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878755.321326:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878755.321333:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9694 00000100:00080000:0.0:1788878755.321340:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878755.321345:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878755.321345:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878755.321353:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878755.321355:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878755.321362:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878755.321365:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878755.321368:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878755.321370:0:177900:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1788878755.321375:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1788878755.321376:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:2.0:1788878755.321384:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788878755.321385:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:2.0:1788878755.321391:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878755.321394:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00100000:2.0:1788878755.321402:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290094976:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878755.321404:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878755.321410:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878755.321413:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878755.321423:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1788878755.321424:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:2.0:1788878755.321428:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878755.321437:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878755.321437:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878755.321445:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878755.321458:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878755.321467:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878755.321475:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878755.321478:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878755.321485:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000008:00100000:0.0:1788878755.321486:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878755.321490:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000008:00000001:0.0:1788878755.321493:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878755.321499:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.321500:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1788878755.321507:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878755.321511:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878755.321513:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878755.321515:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878755.321518:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878755.321528:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878755.321539:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290095104/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878755.321565:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878755.321569:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878755.321605:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788878755.321606:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1788878755.321614:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878755.321623:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878755.321627:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878755.321630:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878755.321633:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878755.321636:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878755.321645:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9694 00000100:00000001:2.0:1788878755.321650:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878755.321657:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.321661:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:0.0:1788878755.321662:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878755.321670:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878755.321672:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878755.321674:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878755.321685:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304367145 00000020:00000001:0.0:1788878755.321686:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878755.321691:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878755.321697:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878755.321702:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878755.321705:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878755.321708:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878755.321710:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1788878755.321717:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878755.321724:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878755.321727:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878755.321734:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878755.321735:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304367145 02000000:00000001:0.0:1788878755.321741:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878755.321745:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878755.321752:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878755.321757:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000008:00000001:0.0:1788878755.321758:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878755.321767:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.321770:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878755.321776:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878755.321778:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878755.321781:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.321786:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788878755.321789:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878755.321794:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878755.321797:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878755.321803:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878755/real 1788878755] req@ffff94e50897ad80 x1875765290094976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878755.321810:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878755.321829:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290095232/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1788878755.321832:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878755.321837:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878755.321849:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878755.321852:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878755.321857:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878755.321860:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.321863:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878755.321868:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.321872:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:0.0:1788878755.321875:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788878755.321879:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788878755.321891:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878755.321897:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1788878755.321899:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290094976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878755.321948:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.321954:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878755.321959:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.321962:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878755.321965:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.321970:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878755.321974:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878755.321976:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788878755.321980:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1788878755.321986:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878755.321993:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00080000:0.0:1788878755.321998:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9694 00000100:00000001:2.0:1788878755.322003:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:0.0:1788878755.322008:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878755.322016:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1788878755.322017:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290094976:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878755.322021:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878755.322034:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788878755.322035:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788878755.322038:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788878755.322045:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1788878755.322046:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878755.322052:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788878755.322054:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:2.0:1788878755.322060:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1788878755.322062:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878755.322068:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788878755.322071:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:0.0:1788878755.322079:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000020:00000001:2.0:1788878755.322079:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878755.322086:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:0.0:1788878755.322088:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878755.322095:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878755.322097:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878755.322101:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878755.322104:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:0.0:1788878755.322111:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878755.322118:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878755.322118:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878755.322126:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1788878755.322127:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878755.322136:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788878755.322138:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878755.322147:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.322155:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1788878755.322157:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878755.322164:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.322165:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878755.322169:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878755.322174:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788878755.322175:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878755.322182:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788878755.322183:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.322189:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878755.322195:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878755.322200:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.322200:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878755.322213:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:0.0:1788878755.322219:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290095360/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788878755.322220:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878755.322232:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878755.322247:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290095104:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878755.322250:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878755.322258:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.322262:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788878755.322268:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878755.322270:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878755.322277:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878755.322281:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878755.322288:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.322301:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878755.322305:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:2.0:1788878755.322314:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878755.322342:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878755.322352:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878755.322357:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878755.322359:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.322363:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878755.322368:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.322375:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304366604 00000100:00000001:2.0:1788878755.322380:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304367145 00000100:00000001:2.0:1788878755.322384:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304366604 00000100:00000001:2.0:1788878755.322387:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304367145 00000100:00000001:2.0:1788878755.322391:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878755.322394:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.322398:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878755.322405:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878755/real 1788878755] req@ffff94e50897bb80 x1875765290095104/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878755.322421:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878755.322424:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878755.322428:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878755.322431:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878755.322434:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.322438:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878755.322440:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878755.322448:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290095104/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878755.322460:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.322463:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878755.322468:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.322471:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878755.322473:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.322476:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878755.322500:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878755.322505:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878755.322512:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878755.322516:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878755.322522:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290095104:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878755.322529:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878755.322531:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878755.322534:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878755.322538:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878755.322541:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878755.322545:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878755.322551:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290095232:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878755.322557:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878755.322561:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878755.322564:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878755.322566:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878755.322590:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.322602:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878755.322605:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:2.0:1788878755.322611:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878755.322634:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878755.322643:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878755.322647:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878755.322649:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.322662:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878755.322667:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.322673:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304366602 00000100:00000001:2.0:1788878755.322678:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304367145 00000100:00000001:2.0:1788878755.322682:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304366602 00000100:00000001:2.0:1788878755.322685:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304367145 00000100:00000001:2.0:1788878755.322689:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878755.322692:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.322696:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878755.322703:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878755/real 1788878755] req@ffff94e508978e00 x1875765290095232/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878755.322719:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878755.322731:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878755.322736:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878755.322739:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878755.322742:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.322745:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878755.322747:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878755.322753:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290095232/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878755.322765:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.322768:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878755.322772:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.322775:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878755.322777:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.322780:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878755.322801:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878755.322806:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878755.322813:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878755.322817:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878755.322822:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290095232:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878755.322829:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.322832:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878755.322833:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878755.322835:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878755.322839:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878755.322840:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878755.322842:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878755.322845:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878755.322847:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878755.322849:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878755.322851:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.322853:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878755.322854:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878755.322856:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878755.322858:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878755.322859:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878755.322870:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878755.322871:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878755.322872:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878755.322875:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878755.322876:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.322878:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.322883:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878755.322884:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.322887:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878755.322889:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878755.322890:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878755.322892:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878755.322894:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878755.322897:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878755.322900:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878755.322930:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878755.322934:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290095360:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878755.322938:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878755.322941:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878755.322943:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878755.322945:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878755.322949:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.322956:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878755.322958:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:2.0:1788878755.322962:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878755.322985:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878755.322991:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878755.322994:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878755.322996:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.322998:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878755.323001:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.323005:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304366603 00000100:00000001:2.0:1788878755.323008:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304367145 00000100:00000001:2.0:1788878755.323010:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304366603 00000100:00000001:2.0:1788878755.323013:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304367145 00000100:00000001:2.0:1788878755.323015:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878755.323017:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.323019:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878755.323024:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878755/real 1788878755] req@ffff94e508978a80 x1875765290095360/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878755.323032:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878755.323033:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878755.323036:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878755.323037:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878755.323039:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.323041:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878755.323043:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878755.323047:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290095360/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878755.323054:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.323056:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878755.323060:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.323061:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878755.323063:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.323065:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878755.323092:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878755.323095:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878755.323099:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878755.323103:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878755.323108:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290095360:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878755.323114:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.323117:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878755.323119:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878755.323122:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878755.323125:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878755.323128:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878755.323130:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878755.323134:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878755.323136:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878755.323139:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878755.323141:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.323144:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878755.323148:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878755.323151:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878755.323154:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878755.323157:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878756.089074:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878756.089081:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878756.089087:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878756.089088:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878756.089111:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878756.089114:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878756.089117:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878757.113094:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878757.113100:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878757.113117:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878757.113118:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878757.113222:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878757.113227:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878757.113230:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878758.137058:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878758.137066:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878758.137071:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878758.137073:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878758.137099:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878758.137102:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878758.137105:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878759.161101:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878759.161107:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878759.161112:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878759.161113:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878759.161171:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878759.161174:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878759.161176:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878760.185016:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878760.185026:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878760.185035:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878760.185038:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878760.185073:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878760.185077:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878760.185083:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878760.441159:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878760.441172:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878760.441175:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878760.441179:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878760.441184:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878760.441192:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9699 00000100:00080000:3.0:1788878760.441205:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878760.441219:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878760.441221:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878760.441230:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878760.441234:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878760.441238:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878760.441241:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878760.441243:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878760.441245:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878760.441250:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878760.441252:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878760.441255:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878760.441259:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878760.441262:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878760.441265:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878760.441267:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878760.441292:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878760.441294:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878760.441297:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878760.441301:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878760.441305:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878760.441319:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878760.441330:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290095488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878760.441353:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878760.441358:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878760.441406:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878760.441410:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878760.441414:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878760.441416:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878760.441419:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878760.441423:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878760.441427:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9699 00000100:00080000:3.0:1788878760.441431:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878760.441434:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878760.441436:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878760.441440:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878760.441443:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878760.441446:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878760.441449:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878760.441452:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878760.441455:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878760.441460:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878760.441464:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878760.441466:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878760.441486:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788878760.441486:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878760.441494:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.441496:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878760.441498:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878760.441504:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1788878760.441509:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878760.441510:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878760.441516:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788878760.441522:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878760.441526:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878760.441531:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878760.441534:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878760.441565:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878760.441570:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:2.0:1788878760.441575:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290095488:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788878760.441594:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878760.441611:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878760.441617:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878760.441618:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878760.441627:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878760.441627:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878760.441634:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878760.441639:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878760.441642:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878760.441645:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878760.441659:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290095616/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878760.441667:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878760.441670:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878760.441673:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:3.0:1788878760.441676:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.441681:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878760.441685:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878760.441691:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878760.441696:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878760.441698:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878760.441700:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878760.441704:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878760.441707:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9699 00000100:00080000:3.0:1788878760.441714:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878760.441724:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878760.441725:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878760.441730:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878760.441732:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878760.441734:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878760.441737:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878760.441741:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000800:00000100:2.0:1788878760.441746:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878760.441747:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878760.441758:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878760.441762:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878760.441764:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878760.441767:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788878760.441769:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878760.441774:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.441776:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878760.441777:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878760.441795:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878760.441796:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788878760.441805:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.441806:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878760.441810:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.441816:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878760.441818:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878760.441822:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.441824:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304372265 00000100:00000001:3.0:1788878760.441825:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878760.441830:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878760.441835:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.441844:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878760.441845:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878760.441857:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290095744/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878760.441860:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304372265 00000100:00000001:2.0:1788878760.441883:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878760.441887:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878760.441891:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878760.441892:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878760.441899:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.441902:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878760.441904:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788878760.441909:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788878760.441911:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878760.441916:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878760.441918:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878760.441959:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878760.441963:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9699 00000100:00080000:3.0:1788878760.441967:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878760.441970:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878760.441972:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878760.441974:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878760.441977:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878760.441979:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878760.441981:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878760.441984:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878760.441986:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878760.441995:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:2.0:1788878760.441997:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878760/real 1788878760] req@ffff94e50449f800 x1875765290095488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878760.441999:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788878760.442014:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878760.442017:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878760.442023:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878760.442027:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878760.442029:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442032:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878760.442035:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878760.442042:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290095488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878760.442052:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442056:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878760.442069:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442072:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878760.442074:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442076:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878760.442079:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878760.442083:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1788878760.442087:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:2.0:1788878760.442094:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 02000000:00000001:3.0:1788878760.442096:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878760.442099:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1788878760.442102:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878760.442109:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:2.0:1788878760.442110:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290095488:192.168.203.129@tcp:250:kworker.0 00000008:00100000:3.0:1788878760.442113:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878760.442121:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442124:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878760.442124:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878760.442137:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878760.442140:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788878760.442141:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442144:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1788878760.442147:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878760.442150:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878760.442151:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878760.442158:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878760.442159:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878760.442164:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:3.0:1788878760.442165:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878760.442167:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878760.442175:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00100000:3.0:1788878760.442180:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290095872/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788878760.442181:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878760.442187:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878760.442192:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878760.442204:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878760.442206:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878760.442211:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878760.442216:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442219:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878760.442224:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442228:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878760.442230:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878760.442232:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878760.442234:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878760.442236:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878760.442240:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878760.442243:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878760.442246:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878760.442252:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290095616:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878760.442256:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878760.442258:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878760.442259:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878760.442267:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878760.442271:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442278:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878760.442279:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:2.0:1788878760.442284:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878760.442299:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878760.442304:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878760.442307:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878760.442308:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442310:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878760.442313:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442318:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304371724 00000100:00000001:2.0:1788878760.442332:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304372265 00000100:00000001:2.0:1788878760.442335:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304371724 00000100:00000001:2.0:1788878760.442337:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304372265 00000100:00000001:2.0:1788878760.442340:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878760.442342:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442344:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878760.442348:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878760/real 1788878760] req@ffff94e50449c700 x1875765290095616/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878760.442376:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878760.442378:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878760.442381:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878760.442383:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878760.442385:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442388:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878760.442389:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878760.442394:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290095616/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878760.442403:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442406:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878760.442409:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442411:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878760.442414:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442416:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878760.442447:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878760.442450:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878760.442454:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878760.442457:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878760.442460:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290095616:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878760.442464:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878760.442466:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878760.442467:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878760.442469:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878760.442471:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878760.442473:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878760.442482:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290095744:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878760.442485:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878760.442488:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878760.442489:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878760.442491:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878760.442494:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442500:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878760.442501:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:2.0:1788878760.442505:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878760.442517:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878760.442521:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878760.442523:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878760.442525:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442527:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878760.442529:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442532:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304371722 00000100:00000001:2.0:1788878760.442535:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304372265 00000100:00000001:2.0:1788878760.442572:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304371722 00000100:00000001:2.0:1788878760.442574:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304372265 00000100:00000001:2.0:1788878760.442577:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878760.442579:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442581:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878760.442586:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878760/real 1788878760] req@ffff94e50449f480 x1875765290095744/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878760.442595:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878760.442597:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878760.442600:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878760.442602:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878760.442603:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442605:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878760.442606:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878760.442611:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290095744/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878760.442618:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442620:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878760.442622:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442624:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878760.442625:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442627:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878760.442647:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878760.442650:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878760.442653:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878760.442655:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878760.442659:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290095744:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878760.442663:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878760.442664:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878760.442666:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878760.442668:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878760.442670:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878760.442672:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878760.442675:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290095872:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878760.442679:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878760.442681:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878760.442683:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878760.442684:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878760.442688:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442692:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878760.442694:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:2.0:1788878760.442697:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878760.442709:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878760.442716:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878760.442718:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878760.442720:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442722:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878760.442724:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442727:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304371722 00000100:00000001:2.0:1788878760.442730:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304372265 00000100:00000001:2.0:1788878760.442733:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304371722 00000100:00000001:2.0:1788878760.442735:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304372265 00000100:00000001:2.0:1788878760.442737:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878760.442739:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442741:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878760.442745:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878760/real 1788878760] req@ffff94e50449d180 x1875765290095872/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878760.442753:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878760.442754:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878760.442756:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878760.442758:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878760.442759:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442761:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878760.442762:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878760.442766:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290095872/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878760.442780:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442782:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878760.442785:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442787:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878760.442788:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442789:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878760.442801:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878760.442804:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878760.442807:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878760.442809:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878760.442812:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290095872:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878760.442816:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442818:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878760.442819:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878760.442821:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878760.442823:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878760.442825:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878760.442827:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878760.442829:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878760.442831:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878760.442833:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878760.442834:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442836:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878760.442837:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878760.442839:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878760.442840:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878760.442842:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878760.442843:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878760.442845:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878760.442846:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878760.442847:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878760.442849:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442850:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878760.442851:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878760.442853:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878760.442855:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878760.442856:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878760.442858:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878760.442859:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878760.442861:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878760.442862:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878760.442864:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442866:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878760.442869:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878760.442870:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878760.442872:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878760.442874:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878761.209006:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878761.209011:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878761.209015:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878761.209018:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878761.209020:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878761.209023:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878761.209025:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878761.209027:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878761.209029:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878761.209030:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878761.209128:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878761.209132:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878761.209135:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878762.233030:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878762.233037:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878762.233042:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878762.233043:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878762.233063:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878762.233066:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878762.233068:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878763.257034:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878763.257040:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878763.257044:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878763.257045:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878763.257089:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878763.257091:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878763.257093:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878764.281026:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878764.281035:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878764.281038:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878764.281041:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878764.281046:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878764.281049:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878764.281051:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878764.281052:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878764.281055:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878764.281057:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878764.281080:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878764.281082:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878764.281084:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878765.305070:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878765.305078:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878765.305086:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878765.305087:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878765.305173:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878765.305176:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878765.305179:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878765.561014:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878765.561036:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878765.561039:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878765.561042:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878765.561047:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878765.561053:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9704 00000100:00080000:3.0:1788878765.561062:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878765.561065:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878765.561067:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878765.561073:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878765.561077:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878765.561080:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878765.561082:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878765.561084:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878765.561085:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878765.561088:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878765.561091:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878765.561094:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878765.561097:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878765.561100:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878765.561102:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878765.561104:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878765.561113:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878765.561115:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878765.561117:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878765.561120:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878765.561125:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878765.561136:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878765.561144:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290096000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878765.561152:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878765.561156:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878765.561174:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878765.561178:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878765.561180:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878765.561182:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878765.561183:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878765.561186:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878765.561188:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9704 00000100:00080000:3.0:1788878765.561191:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878765.561192:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878765.561195:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878765.561198:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878765.561199:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878765.561201:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878765.561203:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878765.561205:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878765.561206:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878765.561210:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878765.561213:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878765.561215:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878765.561219:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788878765.561220:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878765.561224:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.561225:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878765.561226:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878765.561229:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1788878765.561232:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878765.561233:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:3.0:1788878765.561237:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.561238:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878765.561241:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878765.561244:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878765.561246:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878765.561248:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878765.561249:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878765.561255:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878765.561256:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878765.561257:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878765.561261:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878765.561266:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290096000:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878765.561270:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.561271:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878765.561278:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1788878765.561279:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290096128/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788878765.561281:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878765.561287:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878765.561303:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1788878765.561305:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.561323:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878765.561325:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:0.0:1788878765.561329:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878765.561374:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878765.561387:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878765.561392:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878765.561392:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.561396:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.561399:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878765.561402:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.561405:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878765.561406:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878765.561412:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304377385 00000100:00080000:3.0:1788878765.561412:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878765.561414:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878765.561417:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878765.561421:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878765.561424:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788878765.561425:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9704 00000100:00080000:3.0:1788878765.561430:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878765.561437:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878765.561438:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878765.561441:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878765.561442:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878765.561445:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878765.561448:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878765.561452:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304377385 00000100:00000001:3.0:1788878765.561452:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878765.561456:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878765.561460:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788878765.561461:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878765.561467:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.561468:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878765.561469:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878765.561474:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878765.561475:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878765.561477:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1788878765.561479:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878765.561482:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:0.0:1788878765.561486:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878765/real 1788878765] req@ffff94e50449ce00 x1875765290096000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788878765.561486:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878765.561491:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878765.561493:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878765.561523:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000400:00000001:0.0:1788878765.561525:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1788878765.561526:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788878765.561529:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878765.561532:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878765.561535:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:3.0:1788878765.561535:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878765.561539:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.561541:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878765.561544:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.561546:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878765.561577:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878765.561583:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290096000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878765.561592:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.561595:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878765.561598:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.561601:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878765.561602:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.561605:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878765.561607:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878765.561611:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878765.561616:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878765.561618:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878765.561624:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290096000:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878765.561630:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.561633:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878765.561635:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878765.561638:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878765.561643:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878765.561645:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878765.561647:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878765.561650:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878765.561652:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878765.561655:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878765.561656:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.561659:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.561666:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878765.561668:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.561670:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878765.561672:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878765.561674:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878765.561676:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878765.561678:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878765.561680:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878765.561682:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878765.561685:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878765.561688:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290096128:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878765.561692:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878765.561695:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878765.561696:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878765.561698:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878765.561702:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.561709:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878765.561710:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:0.0:1788878765.561715:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878765.561731:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878765.561736:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878765.561739:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878765.561741:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.561743:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878765.561746:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.561747:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.561760:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304376843 00000100:00000001:0.0:1788878765.561766:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304377385 00000100:00100000:3.0:1788878765.561769:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290096256/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878765.561770:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304376843 00000100:00000001:0.0:1788878765.561778:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304377385 00000100:00000001:0.0:1788878765.561787:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878765.561796:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878765.561800:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878765.561801:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788878765.561810:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878765.561814:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878765/real 1788878765] req@ffff94e50449d880 x1875765290096128/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878765.561826:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878765.561828:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878765.561833:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878765.561835:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878765.561838:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.561840:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878765.561843:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878765.561848:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290096128/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878765.561859:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.561863:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878765.561867:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.561869:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878765.561871:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.561875:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878765.561922:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878765.561926:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878765.561932:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878765.561947:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878765.561951:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878765.561954:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878765.561956:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878765.561958:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878765.561961:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878765.561965:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9704 00000100:00080000:3.0:1788878765.561971:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878765.561974:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878765.561977:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878765.561982:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878765.561985:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878765.561989:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:3.0:1788878765.561989:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878765.561999:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:0.0:1788878765.562003:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290096128:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878765.562004:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878765.562008:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878765.562028:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788878765.562028:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878765.562034:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.562035:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878765.562037:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878765.562039:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878765.562040:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878765.562043:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878765.562044:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878765.562048:0:2361:0:(genops.c:1216:class_import_put()) Process entered 02000000:00000001:3.0:1788878765.562049:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878765.562051:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788878765.562052:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:0.0:1788878765.562055:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000008:00100000:3.0:1788878765.562056:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788878765.562058:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000008:00000001:3.0:1788878765.562060:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.562062:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878765.562064:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.562065:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.562068:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878765.562070:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878765.562072:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1788878765.562073:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.562075:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878765.562076:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878765.562079:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878765.562080:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788878765.562082:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.562083:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878765.562087:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878765.562089:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878765.562090:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878765.562091:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878765.562093:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878765.562097:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788878765.562098:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290096384/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788878765.562099:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878765.562109:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290096256:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878765.562116:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788878765.562117:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1788878765.562118:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878765.562121:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878765.562124:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878765.562124:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878765.562129:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878765.562132:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.562137:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878765.562140:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:0.0:1788878765.562143:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878765.562155:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878765.562160:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878765.562162:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878765.562163:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.562164:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878765.562166:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.562168:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304376842 00000100:00000001:0.0:1788878765.562170:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304377385 00000100:00000001:0.0:1788878765.562172:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304376842 00000100:00000001:0.0:1788878765.562173:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304377385 00000100:00000001:0.0:1788878765.562175:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878765.562177:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.562178:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878765.562181:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878765/real 1788878765] req@ffff94e50449c380 x1875765290096256/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878765.562187:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878765.562188:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878765.562190:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878765.562191:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878765.562192:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.562193:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878765.562194:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878765.562197:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290096256/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878765.562201:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.562203:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878765.562204:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.562205:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878765.562206:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.562207:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878765.562216:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878765.562217:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878765.562219:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878765.562221:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878765.562223:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290096256:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878765.562225:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.562226:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878765.562227:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878765.562228:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878765.562230:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878765.562231:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878765.562232:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878765.562233:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878765.562234:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878765.562236:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878765.562237:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.562238:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.562240:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878765.562241:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.562243:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878765.562244:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878765.562244:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878765.562245:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878765.562247:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878765.562248:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878765.562249:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878765.562250:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878765.562252:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290096384:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878765.562254:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878765.562255:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878765.562256:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878765.562257:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878765.562259:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.562262:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878765.562263:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:0.0:1788878765.562266:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878765.562273:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878765.562275:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878765.562276:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878765.562277:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.562279:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878765.562280:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.562282:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304376842 00000100:00000001:0.0:1788878765.562284:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304377385 00000100:00000001:0.0:1788878765.562286:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304376842 00000100:00000001:0.0:1788878765.562287:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304377385 00000100:00000001:0.0:1788878765.562289:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878765.562290:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.562302:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878765.562305:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878765/real 1788878765] req@ffff94e50449fb80 x1875765290096384/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878765.562310:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878765.562311:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878765.562312:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878765.562314:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878765.562315:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.562316:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878765.562317:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878765.562319:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290096384/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878765.562324:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.562325:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878765.562327:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.562328:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878765.562329:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.562330:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878765.562339:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878765.562340:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878765.562342:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878765.562343:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878765.562345:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290096384:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878765.562347:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.562348:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878765.562349:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878765.562350:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878765.562351:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878765.562352:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878765.562353:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878765.562354:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878765.562355:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878765.562367:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878765.562368:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.562369:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.562371:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878765.562372:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.562373:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878765.562374:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878765.625150:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878765.625157:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878765.625160:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878765.625166:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878765.625170:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878765.625175:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.625179:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878765.625182:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878765.625185:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878766.328988:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878766.328994:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878766.329001:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878766.329002:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878766.329023:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878766.329025:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878766.329027:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878766.649009:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878766.649017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878766.649021:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878766.649029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878766.649034:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878766.649039:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878766.649043:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878766.649047:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878766.649050:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878767.352986:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878767.352992:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878767.352996:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878767.352997:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878767.353025:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878767.353027:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878767.353029:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878767.672983:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878767.672988:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878767.672990:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878767.672993:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878767.672996:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878767.672999:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878767.673001:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878767.673003:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878767.673005:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878768.376997:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878768.377002:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878768.377006:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878768.377007:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878768.377024:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878768.377026:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878768.377028:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878768.697013:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878768.697021:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878768.697023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878768.697028:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878768.697031:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878768.697035:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878768.697038:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878768.697041:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878768.697045:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878769.401086:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878769.401114:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878769.401122:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878769.401124:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878769.401198:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878769.401204:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878769.401209:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878769.721081:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878769.721093:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878769.721096:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878769.721103:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878769.721109:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878769.721115:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878769.721119:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878769.721124:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878769.721128:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878770.425090:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878770.425099:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878770.425105:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878770.425107:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878770.425149:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878770.425154:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878770.425158:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878770.681008:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878770.681020:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878770.681023:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878770.681028:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878770.681033:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878770.681043:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9709 00000100:00080000:3.0:1788878770.681056:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878770.681059:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878770.681061:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878770.681071:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878770.681074:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878770.681078:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878770.681081:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878770.681082:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878770.681084:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878770.681088:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878770.681090:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878770.681093:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878770.681097:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878770.681100:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878770.681103:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878770.681106:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878770.681122:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878770.681123:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878770.681127:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878770.681130:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878770.681135:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878770.681166:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878770.681176:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290096512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878770.681185:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878770.681189:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878770.681203:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878770.681206:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878770.681209:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878770.681226:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878770.681228:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878770.681231:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878770.681233:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9709 00000100:00080000:3.0:1788878770.681237:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878770.681239:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878770.681241:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878770.681244:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878770.681246:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878770.681248:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878770.681250:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878770.681253:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878770.681254:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878770.681256:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878770.681258:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878770.681260:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878770.681261:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788878770.681267:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878770.681268:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788878770.681273:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.681274:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878770.681279:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878770.681280:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878770.681284:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788878770.681285:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788878770.681289:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1788878770.681290:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878770.681295:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878770.681296:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878770.681303:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878770.681304:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878770.681308:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878770.681311:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:0.0:1788878770.681312:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290096512:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788878770.681313:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878770.681318:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.681320:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788878770.681325:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878770.681326:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878770.681331:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878770.681335:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1788878770.681336:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290096640/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788878770.681343:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878770.681348:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878770.681351:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878770.681354:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878770.681357:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878770.681360:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878770.681362:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878770.681363:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878770.681366:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878770.681367:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788878770.681371:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9709 00000100:00000001:0.0:1788878770.681372:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00080000:3.0:1788878770.681376:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878770.681379:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878770.681380:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878770.681384:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878770.681387:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878770.681388:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878770.681390:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878770.681391:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878770.681393:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878770.681394:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878770.681397:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878770.681399:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878770.681401:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878770.681403:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878770.681405:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878770.681406:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878770.681408:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878770.681410:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878770.681411:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878770.681417:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878770.681418:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878770.681419:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878770.681420:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878770.681421:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878770.681427:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788878770.681432:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1788878770.681436:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290096768/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878770.681444:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878770.681446:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.681449:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878770.681449:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878770.681455:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878770.681457:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1788878770.681459:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878770.681461:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878770.681463:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878770.681481:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878770.681487:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878770.681487:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878770.681493:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9709 00000100:00080000:3.0:1788878770.681507:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878770.681510:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878770.681511:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878770.681512:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878770.681518:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878770.681521:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304382505 00000020:00000001:3.0:1788878770.681528:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878770.681531:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878770.681533:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878770.681535:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878770.681537:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878770.681539:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788878770.681541:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878770.681547:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878770.681548:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878770.681551:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304382505 02000000:00000001:3.0:1788878770.681552:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878770.681559:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878770.681561:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878770.681562:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000008:00100000:3.0:1788878770.681563:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878770.681569:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.681570:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878770.681571:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.681575:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878770.681579:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878770.681581:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1788878770.681583:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878770.681587:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878770.681589:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1788878770.681591:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878770/real 1788878770] req@ffff94e50449df80 x1875765290096512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788878770.681592:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878770.681600:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878770.681610:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290096896/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1788878770.681613:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878770.681616:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878770.681622:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878770.681623:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878770.681625:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.681628:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878770.681629:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.681633:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.681637:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878770.681639:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878770.681645:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290096512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878770.681652:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.681656:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878770.681659:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.681660:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878770.681662:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.681664:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878770.681666:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878770.681668:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878770.681672:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878770.681674:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878770.681677:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290096512:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878770.681681:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.681683:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878770.681685:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878770.681688:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878770.681692:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878770.681694:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878770.681697:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878770.681701:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878770.681703:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878770.681705:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878770.681707:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.681709:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.681717:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878770.681719:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.681721:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878770.681722:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878770.681724:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878770.681725:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878770.681726:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878770.681728:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878770.681730:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878770.681732:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878770.681734:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290096640:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878770.681737:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878770.681739:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878770.681740:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878770.681742:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878770.681744:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.681749:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878770.681751:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000100:00000001:0.0:1788878770.681755:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878770.681777:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878770.681781:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878770.681783:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878770.681785:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.681786:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878770.681789:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.681793:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304381963 00000100:00000001:0.0:1788878770.681795:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304382505 00000100:00000001:0.0:1788878770.681798:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304381963 00000100:00000001:0.0:1788878770.681800:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304382505 00000100:00000001:0.0:1788878770.681801:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878770.681803:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.681821:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878770.681823:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878770/real 1788878770] req@ffff94e50449e680 x1875765290096640/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878770.681830:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878770.681831:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878770.681833:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878770.681834:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878770.681835:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.681836:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878770.681837:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878770.681840:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290096640/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878770.681845:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.681846:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878770.681848:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.681849:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878770.681849:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.681851:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878770.681864:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878770.681865:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878770.681867:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878770.681869:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878770.681886:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290096640:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878770.681889:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878770.681890:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878770.681891:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878770.681893:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878770.681895:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878770.681896:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878770.681899:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290096768:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878770.681901:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878770.681902:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878770.681904:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878770.681906:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878770.681908:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.681913:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878770.681914:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:0.0:1788878770.681918:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878770.681927:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878770.681955:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878770.681958:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878770.681968:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.681969:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878770.681972:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.681975:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304381962 00000100:00000001:0.0:1788878770.681977:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304382505 00000100:00000001:0.0:1788878770.681979:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304381962 00000100:00000001:0.0:1788878770.681982:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304382505 00000100:00000001:0.0:1788878770.681983:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878770.681985:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.681986:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878770.681990:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878770/real 1788878770] req@ffff94e50449fb80 x1875765290096768/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878770.681996:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878770.681997:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878770.681998:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878770.682000:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878770.682001:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.682003:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878770.682004:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878770.682007:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290096768/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878770.682013:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.682014:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878770.682017:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.682018:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878770.682019:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.682021:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878770.682034:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878770.682037:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878770.682039:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878770.682053:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878770.682057:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290096768:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878770.682062:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878770.682064:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878770.682066:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878770.682068:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878770.682070:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878770.682073:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878770.682077:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290096896:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878770.682081:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878770.682084:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878770.682086:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878770.682088:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878770.682092:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.682099:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878770.682100:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:0.0:1788878770.682105:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878770.682120:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878770.682126:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878770.682128:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878770.682130:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.682131:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878770.682148:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.682151:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304381962 00000100:00000001:0.0:1788878770.682153:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304382505 00000100:00000001:0.0:1788878770.682155:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304381962 00000100:00000001:0.0:1788878770.682157:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304382505 00000100:00000001:0.0:1788878770.682158:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878770.682160:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.682161:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878770.682164:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878770/real 1788878770] req@ffff94e50449c380 x1875765290096896/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878770.682169:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878770.682170:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878770.682172:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878770.682173:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878770.682174:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.682175:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878770.682176:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878770.682179:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290096896/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878770.682184:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.682186:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878770.682187:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.682188:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878770.682189:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.682190:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878770.682219:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878770.682220:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878770.682223:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878770.682224:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878770.682227:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290096896:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878770.682230:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.682232:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878770.682234:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878770.682235:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878770.682237:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878770.682238:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878770.682240:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878770.682242:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878770.682243:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878770.682244:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878770.682246:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.682247:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878770.682248:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878770.682249:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878770.682251:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878770.682252:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878770.682253:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878770.682254:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878770.682255:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878770.682256:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878770.682257:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.682258:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878770.682259:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878770.682260:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878770.682262:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878770.682262:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878770.682263:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878770.682264:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878770.682265:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878770.682266:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878770.682267:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.682269:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.682271:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878770.682273:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.682274:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878770.682275:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878770.744982:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878770.744989:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878770.744991:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878770.744996:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878770.745000:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878770.745004:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.745006:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878770.745009:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878770.745012:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878771.448982:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878771.448988:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878771.448994:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878771.448997:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878771.448999:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878771.449003:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878771.449006:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878771.449008:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878771.449010:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878771.449012:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878771.449042:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878771.449044:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878771.449047:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878771.769053:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878771.769061:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878771.769063:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878771.769070:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878771.769073:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878771.769077:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878771.769080:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878771.769083:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878771.769085:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878772.473030:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878772.473037:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878772.473042:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878772.473044:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878772.473062:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878772.473064:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878772.473066:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878772.793014:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878772.793020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878772.793021:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878772.793026:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878772.793030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878772.793034:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878772.793037:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878772.793039:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878772.793042:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878773.497015:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878773.497021:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878773.497027:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878773.497028:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878773.497151:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878773.497155:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878773.497158:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878773.816983:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878773.816989:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878773.816992:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878773.816997:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878773.817009:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878773.817013:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878773.817016:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878773.817019:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878773.817022:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878774.521034:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878774.521043:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878774.521046:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878774.521049:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878774.521054:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878774.521056:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878774.521058:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878774.521060:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878774.521062:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878774.521064:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878774.521088:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878774.521090:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878774.521092:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878774.840966:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878774.840972:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878774.840974:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878774.840978:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878774.840980:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878774.840983:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878774.840985:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878774.840987:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878774.840989:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878775.545087:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878775.545097:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878775.545104:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878775.545106:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878775.545151:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878775.545153:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878775.545156:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878775.801087:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878775.801108:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878775.801111:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878775.801115:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878775.801119:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878775.801126:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9714 00000100:00080000:2.0:1788878775.801153:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878775.801157:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878775.801160:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878775.801182:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878775.801185:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878775.801190:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878775.801193:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878775.801196:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878775.801198:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878775.801204:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878775.801207:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878775.801211:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878775.801216:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878775.801220:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878775.801223:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878775.801227:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878775.801242:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878775.801244:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878775.801248:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878775.801252:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878775.801257:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878775.801272:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878775.801283:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290097024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878775.801298:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878775.801303:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878775.801323:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878775.801328:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878775.801333:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878775.801335:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878775.801338:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878775.801341:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878775.801345:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9714 00000100:00080000:2.0:1788878775.801351:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878775.801353:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878775.801355:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878775.801374:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878775.801376:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878775.801379:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878775.801382:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878775.801384:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878775.801386:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878775.801390:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878775.801394:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878775.801398:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878775.801403:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878775.801406:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878775.801408:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878775.801422:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878775.801425:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878775.801428:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878775.801448:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878775.801450:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878775.801452:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878775.801454:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878775.801456:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788878775.801457:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.801464:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878775.801468:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878775.801472:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878775.801472:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.801476:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878775.801483:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:2.0:1788878775.801486:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4000 x1875765290097152/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788878775.801487:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878775.801495:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878775.801508:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290097024:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878775.801522:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788878775.801522:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1788878775.801531:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788878775.801531:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878775.801534:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878775.801538:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788878775.801539:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878775.801545:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1788878775.801546:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878775.801555:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878775.801557:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878775.801561:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878775.801565:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878775.801570:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9714 00000100:00000001:0.0:1788878775.801571:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878775.801575:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074559872 : -117763634991744 : ffff94e5045e6d80) 00000100:00080000:2.0:1788878775.801579:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878775.801580:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878775.801586:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878775.801588:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878775.801593:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878775.801595:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878775.801597:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878775.801601:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878775.801604:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878775.801605:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878775.801610:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878775.801613:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878775.801616:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878775.801620:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878775.801623:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788878775.801630:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878775.801630:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878775.801634:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878775.801639:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878775.801641:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.801646:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:2.0:1788878775.801648:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878775.801651:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788878775.801653:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.801655:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.801661:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878775.801662:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878775.801669:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878775.801671:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878775.801673:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.801678:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304387625 00000100:00000001:2.0:1788878775.801682:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878775.801692:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290097280/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878775.801694:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878775.801707:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878775.801708:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304387625 00000100:00000001:2.0:1788878775.801712:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878775.801717:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.801719:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1788878775.801721:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878775.801728:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878775.801729:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878775.801731:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878775.801744:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878775.801747:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9714 00000100:00000001:0.0:1788878775.801753:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788878775.801753:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878775.801759:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.801761:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878775.801766:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878775.801770:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:0.0:1788878775.801772:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878775/real 1788878775] req@ffff94e5045e6d80 x1875765290097024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788878775.801781:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878775.801788:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000001:0.0:1788878775.801792:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1788878775.801794:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000400:00000001:0.0:1788878775.801810:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1788878775.801811:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878775.801816:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878775.801818:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878775.801820:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878775.801822:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878775.801822:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878775.801832:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878775.801836:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878775.801838:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1788878775.801844:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6d80 x1875765290097024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878775.801846:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878775.801856:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878775.801874:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878775.801877:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878775.801885:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.801887:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878775.801895:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878775.801896:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878775.801899:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878775.801902:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.801905:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878775.801907:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878775.801912:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878775.801919:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878775.801922:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878775.801926:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290097024:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1788878775.801944:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878775.801949:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878775.801951:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878775.801957:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878775.801960:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878775.801962:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878775.801965:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878775.801968:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878775.801976:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878775.801984:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290097408/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878775.801994:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802002:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878775.802002:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878775.802007:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788878775.802011:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878775.802012:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878775.802020:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1788878775.802022:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878775.802024:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878775.802031:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878775.802039:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878775.802041:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878775.802045:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878775.802047:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802051:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802060:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878775.802063:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.802066:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878775.802069:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878775.802071:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878775.802073:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878775.802074:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878775.802077:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878775.802080:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878775.802082:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878775.802087:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290097152:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878775.802092:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878775.802110:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878775.802112:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878775.802114:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878775.802117:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.802124:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878775.802126:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074548224 : -117763635003392 : ffff94e5045e4000) 00000100:00000001:0.0:1788878775.802131:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878775.802158:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878775.802180:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878775.802183:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878775.802185:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.802187:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878775.802190:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.802195:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304387084 00000100:00000001:0.0:1788878775.802198:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304387625 00000100:00000001:0.0:1788878775.802200:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304387084 00000100:00000001:0.0:1788878775.802202:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304387625 00000100:00000001:0.0:1788878775.802204:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878775.802205:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.802207:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878775.802211:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878775/real 1788878775] req@ffff94e5045e4000 x1875765290097152/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878775.802219:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878775.802220:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878775.802223:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878775.802224:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878775.802241:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802243:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878775.802245:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878775.802249:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4000 x1875765290097152/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878775.802256:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802258:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878775.802260:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802262:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878775.802263:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802265:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878775.802283:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878775.802286:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878775.802289:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878775.802292:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878775.802295:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290097152:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878775.802299:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878775.802300:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878775.802302:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878775.802304:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878775.802306:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878775.802308:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878775.802311:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290097280:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878775.802314:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878775.802317:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878775.802319:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878775.802321:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878775.802324:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.802330:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878775.802332:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:0.0:1788878775.802337:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878775.802349:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878775.802353:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878775.802355:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878775.802357:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.802358:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878775.802360:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.802374:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304387082 00000100:00000001:0.0:1788878775.802376:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304387625 00000100:00000001:0.0:1788878775.802379:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304387082 00000100:00000001:0.0:1788878775.802381:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304387625 00000100:00000001:0.0:1788878775.802383:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878775.802385:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.802387:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878775.802391:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878775/real 1788878775] req@ffff94e5045e4e00 x1875765290097280/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878775.802397:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878775.802399:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878775.802400:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878775.802402:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878775.802403:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802420:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878775.802421:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878775.802423:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290097280/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878775.802428:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802444:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878775.802447:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802449:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878775.802450:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802451:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878775.802463:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878775.802466:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878775.802468:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878775.802470:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878775.802472:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290097280:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878775.802475:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878775.802476:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878775.802477:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878775.802479:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878775.802480:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878775.802482:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878775.802484:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290097408:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878775.802486:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878775.802487:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878775.802488:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878775.802490:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878775.802492:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.802496:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878775.802497:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00000001:0.0:1788878775.802501:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878775.802508:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878775.802511:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878775.802513:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878775.802514:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.802515:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878775.802517:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.802520:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304387082 00000100:00000001:0.0:1788878775.802522:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304387625 00000100:00000001:0.0:1788878775.802524:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304387082 00000100:00000001:0.0:1788878775.802525:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304387625 00000100:00000001:0.0:1788878775.802527:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878775.802528:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.802529:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878775.802532:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878775/real 1788878775] req@ffff94e5045e7b80 x1875765290097408/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878775.802536:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878775.802537:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878775.802539:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878775.802540:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878775.802541:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802542:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878775.802543:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878775.802545:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290097408/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878775.802549:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802551:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878775.802552:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802553:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878775.802554:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802555:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878775.802566:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878775.802568:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878775.802570:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878775.802572:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878775.802574:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290097408:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878775.802576:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802577:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878775.802579:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878775.802580:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878775.802582:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878775.802583:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878775.802584:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878775.802585:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878775.802586:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878775.802588:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878775.802589:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802590:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878775.802591:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878775.802592:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878775.802594:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878775.802594:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878775.802595:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878775.802597:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878775.802597:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878775.802598:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878775.802599:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802600:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878775.802601:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878775.802601:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878775.802603:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878775.802603:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878775.802604:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878775.802605:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878775.802606:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878775.802606:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878775.802607:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802608:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.802610:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878775.802611:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.802613:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878775.802614:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878775.865057:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878775.865065:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878775.865067:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878775.865072:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878775.865076:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878775.865080:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.865083:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878775.865085:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878775.865088:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878776.569027:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878776.569035:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878776.569041:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878776.569042:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878776.569067:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878776.569070:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878776.569073:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878776.889021:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878776.889030:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878776.889033:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878776.889040:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878776.889045:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878776.889049:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878776.889053:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878776.889056:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878776.889060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878777.593020:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878777.593027:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878777.593032:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878777.593034:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878777.593152:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878777.593157:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878777.593162:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878777.913017:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878777.913025:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878777.913028:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878777.913033:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878777.913038:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878777.913043:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878777.913046:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878777.913049:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878777.913063:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878778.617099:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878778.617110:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878778.617119:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878778.617121:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878778.617154:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878778.617157:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878778.617160:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878778.936989:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878778.936998:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878778.937001:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878778.937008:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878778.937013:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878778.937017:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878778.937021:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878778.937024:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878778.937027:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878779.640990:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878779.640996:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878779.641001:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878779.641002:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878779.641022:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878779.641025:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878779.641027:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878779.960999:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878779.961006:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878779.961009:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878779.961015:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878779.961018:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878779.961022:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878779.961035:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878779.961038:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878779.961041:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878780.664990:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878780.664997:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878780.665002:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878780.665003:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878780.665024:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878780.665026:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878780.665029:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878780.921089:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878780.921098:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878780.921101:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878780.921105:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878780.921110:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878780.921117:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9719 00000100:00080000:0.0:1788878780.921128:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878780.921131:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878780.921133:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878780.921141:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878780.921143:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878780.921147:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878780.921149:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878780.921151:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878780.921153:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878780.921156:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878780.921159:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878780.921161:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878780.921165:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878780.921194:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.921197:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878780.921199:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878780.921211:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878780.921212:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.921216:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878780.921219:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878780.921224:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.921236:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878780.921246:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978700 x1875765290097536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878780.921258:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878780.921262:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.921287:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878780.921291:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878780.921294:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878780.921296:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878780.921298:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878780.921301:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878780.921304:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9719 00000100:00080000:0.0:1788878780.921326:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878780.921329:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878780.921331:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878780.921336:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878780.921338:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878780.921341:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788878780.921344:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878780.921347:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878780.921350:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878780.921365:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878780.921368:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878780.921370:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878780.921377:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878780.921378:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878780.921383:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1788878780.921385:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878780.921387:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788878780.921414:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.921416:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.921419:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878780.921431:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788878780.921432:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878780.921456:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788878780.921457:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878780.921465:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.921466:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878780.921476:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:2.0:1788878780.921479:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290097536:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788878780.921480:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.921486:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878780.921490:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878780.921503:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878780.921506:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878780.921510:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878780.921514:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.921515:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878780.921520:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1788878780.921527:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290097664/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788878780.921528:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.921550:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878780.921551:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878780.921557:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.921557:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00000001:2.0:1788878780.921566:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788878780.921567:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878780.921572:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878780.921577:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878780.921579:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878780.921581:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878780.921585:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878780.921588:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9719 00000100:00080000:0.0:1788878780.921594:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878780.921596:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878780.921598:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878780.921602:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878780.921603:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878780.921606:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878780.921608:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878780.921613:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000800:00000100:2.0:1788878780.921615:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878780.921617:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878780.921626:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878780.921629:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878780.921632:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878780.921634:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:0.0:1788878780.921636:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878780.921641:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1788878780.921642:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.921647:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.921648:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878780.921653:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878780.921655:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000008:00000001:0.0:1788878780.921661:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.921664:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.921665:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878780.921681:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878780.921683:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.921683:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304392745 00000100:00000001:0.0:1788878780.921692:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878780.921694:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878780.921696:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.921702:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878780.921703:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.921716:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304392745 00000100:00100000:0.0:1788878780.921717:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897aa00 x1875765290097792/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878780.921725:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878780.921732:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878780.921739:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.921764:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878780.921780:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878780/real 1788878780] req@ffff94e508978700 x1875765290097536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878780.921782:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878780.921790:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.921806:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878780.921815:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878780.921826:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878780.921832:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878780.921836:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:2.0:1788878780.921839:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788878780.921840:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000400:00000001:2.0:1788878780.921850:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:0.0:1788878780.921852:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9719 00000100:00080000:0.0:1788878780.921863:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878780.921867:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878780.921868:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878780.921876:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878780.921880:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000020:00000001:0.0:1788878780.921881:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878780.921888:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788878780.921890:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788878780.921892:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878780.921905:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1788878780.921906:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878780.921915:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878780.921928:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290097536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878780.921972:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878780.921996:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922018:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878780.922024:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922027:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878780.922030:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922034:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878780.922038:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878780.922044:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878780.922052:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878780.922058:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878780.922064:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290097536:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878780.922084:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878780.922085:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878780.922093:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1788878780.922098:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878780.922099:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878780.922106:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1788878780.922107:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878780.922112:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878780.922112:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:0.0:1788878780.922118:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1788878780.922118:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878780.922124:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878780.922126:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.922130:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:2.0:1788878780.922131:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000008:00100000:0.0:1788878780.922134:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788878780.922135:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000008:00000001:0.0:1788878780.922143:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922143:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878780.922149:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878780.922150:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922155:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788878780.922161:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878780.922166:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922167:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878780.922172:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.922174:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878780.922177:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788878780.922178:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878780.922181:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788878780.922184:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922185:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878780.922191:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878780.922193:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.922196:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878780.922196:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878780.922198:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878780.922205:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878780.922210:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978700 x1875765290097920/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788878780.922216:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290097664:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878780.922233:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788878780.922237:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788878780.922241:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788878780.922242:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878780.922248:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.922250:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878780.922256:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878780.922266:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922279:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878780.922282:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:2.0:1788878780.922289:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878780.922313:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878780.922320:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878780.922324:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878780.922327:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922330:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878780.922334:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922340:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304392204 00000100:00000001:2.0:1788878780.922344:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304392745 00000100:00000001:2.0:1788878780.922346:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304392204 00000100:00000001:2.0:1788878780.922349:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304392745 00000100:00000001:2.0:1788878780.922352:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878780.922354:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922356:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878780.922362:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878780/real 1788878780] req@ffff94e50897a300 x1875765290097664/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878780.922372:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878780.922374:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878780.922377:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878780.922380:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878780.922381:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922384:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878780.922386:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878780.922391:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290097664/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878780.922422:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922426:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878780.922429:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922431:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878780.922433:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922436:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878780.922449:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878780.922452:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878780.922457:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878780.922459:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878780.922463:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290097664:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878780.922467:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878780.922469:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878780.922470:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878780.922473:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878780.922475:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878780.922477:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878780.922480:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290097792:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878780.922484:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878780.922486:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878780.922488:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878780.922500:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878780.922504:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922510:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878780.922512:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:2.0:1788878780.922516:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878780.922530:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878780.922535:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878780.922537:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878780.922539:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922540:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878780.922543:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922546:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304392202 00000100:00000001:2.0:1788878780.922549:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304392745 00000100:00000001:2.0:1788878780.922552:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304392202 00000100:00000001:2.0:1788878780.922554:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304392745 00000100:00000001:2.0:1788878780.922558:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878780.922560:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922562:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878780.922566:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878780/real 1788878780] req@ffff94e50897aa00 x1875765290097792/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878780.922574:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878780.922576:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878780.922578:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878780.922580:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878780.922581:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922583:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878780.922584:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878780.922588:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290097792/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878780.922595:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922597:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878780.922600:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922601:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878780.922603:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922604:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878780.922616:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878780.922618:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878780.922621:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878780.922623:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878780.922626:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290097792:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878780.922630:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922632:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878780.922633:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878780.922635:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878780.922637:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878780.922639:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878780.922641:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878780.922643:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878780.922645:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878780.922648:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878780.922649:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922651:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878780.922652:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878780.922654:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878780.922655:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878780.922657:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878780.922658:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878780.922660:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878780.922661:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878780.922663:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878780.922664:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922666:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922669:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878780.922670:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922672:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878780.922674:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878780.922675:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878780.922676:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878780.922677:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878780.922680:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878780.922681:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878780.922689:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878780.922692:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290097920:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878780.922695:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878780.922697:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878780.922699:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878780.922700:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878780.922702:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922707:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878780.922708:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00000001:2.0:1788878780.922711:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878780.922721:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878780.922725:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878780.922727:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878780.922729:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922731:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878780.922733:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922736:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304392202 00000100:00000001:2.0:1788878780.922739:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304392745 00000100:00000001:2.0:1788878780.922741:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304392202 00000100:00000001:2.0:1788878780.922743:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304392745 00000100:00000001:2.0:1788878780.922745:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878780.922747:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922749:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878780.922752:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878780/real 1788878780] req@ffff94e508978700 x1875765290097920/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878780.922760:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878780.922761:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878780.922763:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878780.922765:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878780.922766:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922768:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878780.922770:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878780.922773:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290097920/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878780.922780:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922782:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878780.922784:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922785:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878780.922787:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922788:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878780.922817:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878780.922821:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878780.922826:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878780.922830:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878780.922835:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290097920:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878780.922841:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922844:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878780.922846:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878780.922849:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878780.922852:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878780.922854:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878780.922856:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878780.922859:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878780.922862:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878780.922865:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878780.922867:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922870:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878780.922875:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878780.922877:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878780.922880:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878780.922883:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878780.985021:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878780.985031:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878780.985034:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878780.985040:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878780.985046:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878780.985051:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878780.985065:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878780.985068:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878780.985071:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878781.688957:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878781.688963:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878781.688967:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878781.688969:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878781.688971:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878781.688973:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878781.688975:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878781.688977:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878781.688978:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878781.688980:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878781.689021:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878781.689025:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878781.689027:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878782.008988:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878782.008996:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878782.008998:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878782.009005:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878782.009008:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878782.009013:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878782.009016:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878782.009018:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878782.009020:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878782.712982:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878782.712987:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878782.712991:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878782.712993:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878782.713008:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878782.713010:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878782.713012:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878783.033007:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878783.033014:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878783.033015:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878783.033019:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878783.033022:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878783.033025:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878783.033027:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878783.033030:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878783.033032:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878783.737029:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878783.737036:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878783.737043:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878783.737045:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878783.737111:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878783.737115:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878783.737120:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878784.056979:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878784.056986:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878784.056988:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878784.056993:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878784.056997:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878784.057000:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878784.057002:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878784.057005:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878784.057007:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878784.761005:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878784.761012:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878784.761015:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878784.761018:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878784.761022:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878784.761024:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878784.761026:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878784.761028:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878784.761031:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878784.761032:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878784.761062:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878784.761064:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878784.761066:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878785.081019:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878785.081031:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878785.081035:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878785.081045:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878785.081052:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878785.081059:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878785.081064:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878785.081068:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878785.081074:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878785.785008:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878785.785015:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878785.785021:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878785.785022:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878785.785104:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878785.785109:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878785.785113:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878786.041020:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878786.041036:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878786.041041:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878786.041047:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878786.041056:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878786.041067:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9724 00000100:00080000:3.0:1788878786.041084:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878786.041089:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878786.041092:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878786.041102:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878786.041105:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878786.041110:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878786.041113:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878786.041117:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878786.041119:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878786.041126:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878786.041132:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878786.041138:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878786.041144:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878786.041150:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878786.041155:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878786.041159:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878786.041174:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878786.041177:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878786.041183:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878786.041187:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878786.041195:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878786.041214:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878786.041228:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290098048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878786.041243:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878786.041248:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878786.041270:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878786.041274:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878786.041279:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878786.041281:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878786.041283:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878786.041287:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878786.041290:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9724 00000100:00080000:3.0:1788878786.041296:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878786.041299:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878786.041301:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878786.041307:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878786.041310:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878786.041314:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878786.041318:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878786.041321:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878786.041323:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878786.041328:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878786.041331:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.041335:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788878786.041335:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878786.041345:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878786.041346:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788878786.041351:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.041355:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878786.041357:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878786.041388:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1788878786.041388:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878786.041395:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878786.041398:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878786.041404:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878786.041406:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.041411:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878786.041412:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878786.041416:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878786.041420:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878786.041428:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878786.041431:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878786.041435:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788878786.041444:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290098176/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788878786.041445:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878786.041463:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290098048:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878786.041473:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878786.041475:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878786.041483:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878786.041485:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878786.041489:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878786.041495:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878786.041500:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878786.041504:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878786.041510:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878786.041521:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878786.041524:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788878786.041528:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788878786.041529:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878786.041534:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:3.0:1788878786.041540:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788878786.041543:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788878786.041554:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9724 00000100:00080000:3.0:1788878786.041564:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878786.041567:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878786.041570:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878786.041589:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878786.041592:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878786.041596:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878786.041599:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878786.041602:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878786.041606:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:2.0:1788878786.041610:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1788878786.041618:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878786.041624:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878786.041627:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878786.041632:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788878786.041633:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878786.041641:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.041644:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878786.041645:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878786.041653:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788878786.041654:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878786.041661:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878786.041663:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.041671:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788878786.041671:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878786.041677:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.041681:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878786.041681:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878786.041686:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878786.041692:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304397865 02000000:00000001:3.0:1788878786.041693:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878786.041710:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.041714:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00100000:3.0:1788878786.041721:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290098304/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878786.041726:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304397865 00000100:00000001:2.0:1788878786.041740:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878786.041743:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878786.041748:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878786.041752:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878786.041755:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788878786.041756:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788878786.041768:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878786.041770:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.041774:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878786.041783:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878786/real 1788878786] req@ffff94e50449d880 x1875765290098048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878786.041788:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878786.041795:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878786.041810:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000400:00000001:2.0:1788878786.041812:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878786.041818:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1788878786.041825:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9724 00000100:00000001:2.0:1788878786.041826:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878786.041834:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878786.041839:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788878786.041841:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878786.041844:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878786.041850:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878786.041853:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878786.041862:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:2.0:1788878786.041863:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290098048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788878786.041877:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788878786.041882:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788878786.041883:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878786.041891:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878786.041893:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1788878786.041899:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878786.041942:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.041949:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878786.041952:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878786.041954:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878786.041958:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878786.041960:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878786.041969:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878786.041974:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00080000:2.0:1788878786.041977:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788878786.041983:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878786.041992:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:2.0:1788878786.041993:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878786.042003:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1788878786.042005:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878786.042017:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878786.042022:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878786.042026:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788878786.042028:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290098048:192.168.203.129@tcp:250:kworker.0 00000008:00000001:3.0:1788878786.042032:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878786.042042:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042050:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878786.042056:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878786.042059:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788878786.042062:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042067:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1788878786.042071:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878786.042075:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788878786.042079:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878786.042085:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:3.0:1788878786.042087:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878786.042094:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:3.0:1788878786.042099:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878786.042101:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878786.042112:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00100000:3.0:1788878786.042114:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290098432/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788878786.042116:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878786.042125:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878786.042130:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042137:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878786.042148:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878786.042151:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878786.042157:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042158:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042167:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878786.042173:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042174:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878786.042179:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878786.042184:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878786.042187:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878786.042192:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878786.042195:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878786.042198:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878786.042205:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290098176:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878786.042210:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878786.042214:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878786.042216:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878786.042219:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878786.042223:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042235:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878786.042239:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:2.0:1788878786.042246:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878786.042274:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878786.042284:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878786.042289:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878786.042292:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042296:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878786.042302:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042310:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304397324 00000100:00000001:2.0:1788878786.042316:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304397865 00000100:00000001:2.0:1788878786.042322:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304397324 00000100:00000001:2.0:1788878786.042326:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304397865 00000100:00000001:2.0:1788878786.042331:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878786.042335:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042339:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878786.042348:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878786/real 1788878786] req@ffff94e50449ce00 x1875765290098176/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878786.042386:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878786.042389:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878786.042394:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878786.042396:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878786.042398:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042401:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878786.042403:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878786.042409:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290098176/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878786.042419:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042422:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878786.042426:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042428:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878786.042429:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042431:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878786.042446:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878786.042449:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878786.042453:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878786.042456:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878786.042461:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290098176:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878786.042466:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878786.042468:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878786.042470:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878786.042473:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878786.042475:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878786.042478:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878786.042482:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290098304:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878786.042486:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878786.042489:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878786.042491:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878786.042493:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878786.042498:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042505:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878786.042507:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:2.0:1788878786.042512:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878786.042530:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878786.042537:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878786.042539:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878786.042541:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042544:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878786.042547:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042551:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304397322 00000100:00000001:2.0:1788878786.042555:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304397865 00000100:00000001:2.0:1788878786.042557:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304397322 00000100:00000001:2.0:1788878786.042560:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304397865 00000100:00000001:2.0:1788878786.042563:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878786.042566:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042568:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878786.042573:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878786/real 1788878786] req@ffff94e50449d180 x1875765290098304/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878786.042591:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878786.042593:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878786.042596:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878786.042598:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878786.042599:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042601:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878786.042603:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878786.042607:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290098304/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878786.042615:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042618:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878786.042620:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042622:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878786.042624:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042627:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878786.042638:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878786.042642:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878786.042646:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878786.042649:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878786.042653:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290098304:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878786.042657:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878786.042659:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878786.042661:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878786.042663:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878786.042665:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878786.042667:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878786.042671:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290098432:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878786.042674:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878786.042676:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878786.042678:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878786.042681:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878786.042684:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042689:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878786.042691:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:2.0:1788878786.042695:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878786.042706:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878786.042711:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878786.042713:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878786.042715:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042717:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878786.042720:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042724:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304397322 00000100:00000001:2.0:1788878786.042727:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304397865 00000100:00000001:2.0:1788878786.042730:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304397322 00000100:00000001:2.0:1788878786.042733:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304397865 00000100:00000001:2.0:1788878786.042735:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878786.042737:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042739:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878786.042744:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878786/real 1788878786] req@ffff94e50449f480 x1875765290098432/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878786.042753:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878786.042754:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878786.042757:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878786.042758:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878786.042760:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042762:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878786.042764:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878786.042768:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290098432/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878786.042776:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042779:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878786.042782:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042784:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878786.042786:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042788:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878786.042800:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878786.042803:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878786.042807:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878786.042810:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878786.042814:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290098432:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878786.042818:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042820:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878786.042822:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878786.042824:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878786.042826:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878786.042828:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878786.042830:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878786.042833:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878786.042834:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878786.042837:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878786.042839:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042841:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878786.042842:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878786.042844:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878786.042846:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878786.042847:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878786.042849:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878786.042851:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878786.042852:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878786.042854:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878786.042855:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042857:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878786.042858:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878786.042860:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878786.042862:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878786.042864:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878786.042865:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878786.042867:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878786.042869:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878786.042871:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878786.042873:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042875:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878786.042880:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878786.042882:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878786.042884:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878786.042886:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878786.105031:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878786.105037:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878786.105040:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878786.105045:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878786.105049:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878786.105053:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878786.105056:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878786.105059:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878786.105062:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878786.809003:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878786.809009:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878786.809015:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878786.809016:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878786.809036:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878786.809039:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878786.809042:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878787.129034:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878787.129042:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878787.129045:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878787.129051:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878787.129056:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878787.129060:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878787.129064:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878787.129067:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878787.129071:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878787.833061:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878787.833072:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878787.833081:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878787.833084:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878787.833120:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878787.833125:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878787.833131:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878788.153053:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878788.153064:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878788.153068:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878788.153078:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878788.153085:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878788.153091:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878788.153097:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878788.153101:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878788.153106:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878788.856989:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878788.856997:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878788.857003:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878788.857005:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878788.857027:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878788.857030:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878788.857032:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878789.177007:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878789.177014:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878789.177016:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878789.177020:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878789.177023:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878789.177027:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878789.177029:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878789.177031:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878789.177034:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878789.881039:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878789.881046:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878789.881052:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878789.881054:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878789.881135:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878789.881140:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878789.881174:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878790.201004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878790.201011:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878790.201014:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878790.201019:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878790.201023:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878790.201028:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878790.201031:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878790.201034:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878790.201037:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878790.904990:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878790.904998:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878790.905005:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878790.905008:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878790.905036:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878790.905039:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878790.905041:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878791.161063:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878791.161079:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878791.161084:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878791.161106:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878791.161114:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878791.161126:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9730 00000100:00080000:3.0:1788878791.161142:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878791.161148:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878791.161152:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878791.161166:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878791.161170:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878791.161193:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878791.161215:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878791.161220:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878791.161222:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878791.161229:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878791.161232:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878791.161236:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878791.161240:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878791.161243:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878791.161246:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878791.161249:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878791.161263:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878791.161265:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878791.161269:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878791.161272:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878791.161279:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878791.161296:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878791.161310:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290098560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878791.161348:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878791.161358:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878791.161382:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878791.161388:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878791.161395:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878791.161398:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878791.161401:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878791.161405:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878791.161409:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9730 00000100:00080000:3.0:1788878791.161415:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878791.161419:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878791.161421:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878791.161426:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878791.161429:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878791.161432:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878791.161435:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878791.161437:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878791.161439:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878791.161444:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878791.161447:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878791.161449:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878791.161451:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788878791.161458:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878791.161461:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788878791.161464:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878791.161469:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878791.161470:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1788878791.161474:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878791.161476:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878791.161482:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788878791.161483:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878791.161491:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878791.161493:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878791.161498:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878791.161499:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878791.161504:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878791.161506:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878791.161509:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878791.161513:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878791.161518:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878791.161519:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290098560:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878791.161527:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.161529:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878791.161537:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878791.161543:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878791.161546:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290098688/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788878791.161547:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878791.161556:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878791.161571:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878791.161577:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.161580:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878791.161585:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:3.0:1788878791.161586:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.161592:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788878791.161595:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878791.161604:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878791.161607:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878791.161611:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878791.161616:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878791.161622:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9730 00000100:00080000:3.0:1788878791.161630:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878791.161636:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878791.161639:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:2.0:1788878791.161642:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:3.0:1788878791.161646:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878791.161651:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878791.161654:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878791.161658:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878791.161661:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878791.161662:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878791.161668:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878791.161672:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1788878791.161677:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878791.161679:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878791.161683:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.161684:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788878791.161688:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878791.161694:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878791.161694:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878791.161701:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.161703:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304402985 00000100:00000001:3.0:1788878791.161704:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878791.161711:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878791.161715:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878791.161718:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.161720:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788878791.161726:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878791.161731:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304402985 02000000:00000001:3.0:1788878791.161732:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878791.161740:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878791.161743:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878791.161745:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1788878791.161747:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878791.161758:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.161765:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788878791.161766:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290098816/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878791.161777:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.161786:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788878791.161788:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878791.161793:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878791.161799:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878791/real 1788878791] req@ffff94e50449c700 x1875765290098560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878791.161801:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878791.161810:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878791.161821:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878791.161825:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878791.161832:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:2.0:1788878791.161834:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788878791.161836:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000400:00000001:2.0:1788878791.161840:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1788878791.161845:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9730 00000100:00000001:2.0:1788878791.161849:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:3.0:1788878791.161852:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878791.161855:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878791.161857:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878791.161861:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878791.161862:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878791.161871:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1788878791.161872:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788878791.161878:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788878791.161878:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878791.161883:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878791.161888:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:2.0:1788878791.161891:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290098560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878791.161893:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878791.161897:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878791.161951:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878791.161962:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.161963:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878791.161965:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878791.161972:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878791.161976:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878791.161977:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878791.161983:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878791.161988:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1788878791.161989:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878791.161998:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162000:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878791.162003:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162008:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878791.162010:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878791.162016:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162017:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878791.162019:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878791.162025:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878791.162027:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878791.162029:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878791.162039:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878791.162041:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788878791.162065:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290098944/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878791.162075:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878791.162085:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878791.162099:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290098560:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878791.162099:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878791.162110:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162114:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162121:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878791.162122:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162124:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878791.162131:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878791.162138:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878791.162156:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878791.162159:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878791.162184:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878791.162187:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878791.162191:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878791.162193:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162196:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162205:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878791.162208:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162212:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878791.162215:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878791.162218:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878791.162220:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878791.162223:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878791.162227:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878791.162231:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878791.162235:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878791.162242:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290098688:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878791.162249:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878791.162252:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878791.162255:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878791.162259:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878791.162265:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162277:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878791.162280:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:2.0:1788878791.162288:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878791.162309:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878791.162316:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878791.162319:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878791.162321:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162323:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878791.162349:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162356:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304402444 00000100:00000001:2.0:1788878791.162360:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304402985 00000100:00000001:2.0:1788878791.162362:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304402444 00000100:00000001:2.0:1788878791.162365:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304402985 00000100:00000001:2.0:1788878791.162368:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878791.162370:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162373:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878791.162380:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878791/real 1788878791] req@ffff94e50449f800 x1875765290098688/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878791.162398:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878791.162401:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878791.162406:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878791.162409:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878791.162412:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162415:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878791.162418:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878791.162425:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290098688/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878791.162441:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162445:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878791.162450:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162454:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878791.162456:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162459:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878791.162478:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878791.162482:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878791.162487:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878791.162490:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878791.162494:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290098688:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878791.162499:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878791.162501:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878791.162503:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878791.162506:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878791.162508:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878791.162510:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878791.162514:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290098816:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878791.162521:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878791.162524:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878791.162526:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878791.162528:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878791.162532:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162540:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878791.162542:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:2.0:1788878791.162546:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878791.162562:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878791.162568:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878791.162571:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878791.162573:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162575:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878791.162578:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162582:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304402442 00000100:00000001:2.0:1788878791.162585:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304402985 00000100:00000001:2.0:1788878791.162588:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304402442 00000100:00000001:2.0:1788878791.162590:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304402985 00000100:00000001:2.0:1788878791.162593:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878791.162595:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162597:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878791.162602:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878791/real 1788878791] req@ffff94e50449e300 x1875765290098816/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878791.162610:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878791.162612:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878791.162614:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878791.162616:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878791.162618:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162620:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878791.162622:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878791.162626:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290098816/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878791.162642:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162644:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878791.162664:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162666:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878791.162668:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162670:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878791.162683:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878791.162686:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878791.162689:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878791.162692:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878791.162695:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290098816:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878791.162700:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878791.162702:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878791.162704:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878791.162707:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878791.162709:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878791.162711:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878791.162715:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290098944:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878791.162718:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878791.162720:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878791.162722:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878791.162724:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878791.162727:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162733:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878791.162734:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:2.0:1788878791.162739:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878791.162751:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878791.162755:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878791.162758:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878791.162760:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162762:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878791.162765:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162787:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304402442 00000100:00000001:2.0:1788878791.162790:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304402985 00000100:00000001:2.0:1788878791.162792:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304402442 00000100:00000001:2.0:1788878791.162795:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304402985 00000100:00000001:2.0:1788878791.162797:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878791.162799:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162801:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878791.162806:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878791/real 1788878791] req@ffff94e50449f100 x1875765290098944/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878791.162814:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878791.162815:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878791.162818:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878791.162820:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878791.162821:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162823:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878791.162824:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878791.162828:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290098944/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878791.162836:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162839:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878791.162841:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162843:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878791.162845:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162847:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878791.162858:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878791.162860:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878791.162863:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878791.162866:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878791.162870:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290098944:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878791.162892:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162895:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878791.162897:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878791.162899:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878791.162902:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878791.162903:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878791.162905:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878791.162908:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878791.162909:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878791.162912:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878791.162914:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162916:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878791.162918:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878791.162919:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878791.162922:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878791.162923:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878791.162924:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878791.162926:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878791.162928:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878791.162930:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878791.162931:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162933:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878791.162934:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878791.162936:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878791.162937:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878791.162939:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878791.162940:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878791.162942:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878791.162943:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878791.162944:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878791.162946:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162948:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878791.162992:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878791.162995:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878791.162999:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878791.163001:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878791.225037:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878791.225044:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878791.225047:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878791.225053:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878791.225057:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878791.225062:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878791.225065:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878791.225068:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878791.225071:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878791.929028:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878791.929034:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878791.929037:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878791.929040:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878791.929042:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878791.929044:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878791.929046:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878791.929048:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878791.929050:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878791.929052:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878791.929071:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878791.929074:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878791.929076:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878792.249054:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878792.249062:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878792.249064:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878792.249069:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878792.249073:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878792.249077:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878792.249080:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878792.249083:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878792.249085:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878792.953035:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878792.953041:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878792.953046:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878792.953048:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878792.953069:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878792.953072:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878792.953075:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878793.273046:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878793.273056:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878793.273059:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878793.273066:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878793.273071:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878793.273076:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878793.273079:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878793.273082:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878793.273086:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878793.977095:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878793.977105:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878793.977114:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878793.977116:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878793.977212:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878793.977218:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878793.977222:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878794.297040:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878794.297050:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878794.297053:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878794.297061:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878794.297067:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878794.297073:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878794.297077:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878794.297080:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878794.297085:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878795.001018:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878795.001027:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878795.001030:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878795.001034:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878795.001040:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878795.001043:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878795.001046:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878795.001049:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878795.001053:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878795.001055:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878795.001080:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878795.001083:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878795.001087:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878795.321010:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878795.321018:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878795.321021:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878795.321027:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878795.321032:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878795.321037:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878795.321041:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878795.321044:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878795.321047:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878796.025015:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878796.025022:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878796.025029:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878796.025031:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878796.025066:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878796.025069:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878796.025073:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878796.281093:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878796.281104:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878796.281108:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878796.281113:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878796.281119:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878796.281127:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9735 00000100:00080000:2.0:1788878796.281140:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878796.281144:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878796.281146:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878796.281156:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878796.281159:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878796.281164:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878796.281167:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878796.281170:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878796.281172:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878796.281177:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878796.281180:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878796.281184:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878796.281188:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878796.281192:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878796.281195:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878796.281198:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878796.281210:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878796.281212:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878796.281216:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878796.281220:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878796.281226:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878796.281239:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878796.281251:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290099072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878796.281265:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878796.281271:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878796.281324:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878796.281329:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878796.281337:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878796.281339:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878796.281342:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878796.281346:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878796.281352:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9735 00000100:00080000:2.0:1788878796.281357:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878796.281359:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878796.281362:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878796.281366:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878796.281368:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878796.281371:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878796.281374:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788878796.281374:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878796.281382:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878796.281383:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878796.281385:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878796.281390:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1788878796.281395:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878796.281397:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878796.281402:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.281403:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878796.281409:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878796.281414:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878796.281420:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878796.281421:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878796.281432:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878796.281433:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878796.281439:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878796.281446:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1788878796.281448:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290099072:192.168.203.129@tcp:250:kworker.0 00000008:00000001:2.0:1788878796.281454:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.281462:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788878796.281463:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878796.281469:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878796.281471:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878796.281475:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878796.281476:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878796.281484:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788878796.281485:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878796.281490:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878796.281493:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878796.281494:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878796.281503:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878796.281515:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290099200/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878796.281519:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878796.281525:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00000001:0.0:1788878796.281536:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878796.281546:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878796.281550:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878796.281555:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878796.281558:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878796.281562:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878796.281564:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878796.281567:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878796.281571:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878796.281575:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9735 00000100:00080000:2.0:1788878796.281580:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878796.281582:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878796.281584:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:0.0:1788878796.281594:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:2.0:1788878796.281595:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878796.281602:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878796.281604:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878796.281609:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878796.281613:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878796.281614:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878796.281622:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878796.281623:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878796.281628:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.281634:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:2.0:1788878796.281641:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878796.281643:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.281655:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304408105 00000100:00000001:2.0:1788878796.281655:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878796.281666:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878796.281675:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1788878796.281677:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878796.281687:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304408105 02000000:00000001:2.0:1788878796.281687:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878796.281699:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878796.281703:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000008:00100000:2.0:1788878796.281707:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878796.281716:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878796.281720:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.281728:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878796.281729:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878796.281735:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.281740:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878796.281743:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878796.281751:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1788878796.281751:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878796.281760:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878796.281772:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878796/real 1788878796] req@ffff94e5045e7480 x1875765290099072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878796.281775:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878796.281797:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290099328/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1788878796.281806:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878796.281814:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878796.281825:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878796.281826:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878796.281831:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878796.281837:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.281838:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.281849:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878796.281853:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.281854:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878796.281865:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788878796.281873:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290099072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878796.281875:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878796.281883:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878796.281889:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878796.281899:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878796.281947:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.281957:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878796.281959:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9735 00000100:00000001:0.0:1788878796.281966:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878796.281971:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878796.281972:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878796.281977:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878796.281979:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878796.281982:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878796.281985:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878796.281987:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1788878796.281994:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00080000:0.0:1788878796.281996:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:2.0:1788878796.281999:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878796.282005:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00080000:0.0:1788878796.282006:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000020:00000001:2.0:1788878796.282014:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878796.282016:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788878796.282021:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1788878796.282028:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290099072:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878796.282029:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878796.282045:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878796.282046:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878796.282056:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878796.282057:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282062:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878796.282063:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878796.282068:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878796.282071:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878796.282075:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:2.0:1788878796.282078:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878796.282081:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:2.0:1788878796.282083:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000020:00000001:0.0:1788878796.282087:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000008:00100000:2.0:1788878796.282089:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788878796.282095:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000008:00000001:2.0:1788878796.282097:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878796.282100:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878796.282102:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878796.282110:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878796.282111:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878796.282113:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878796.282115:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878796.282121:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282123:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878796.282128:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878796.282131:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282132:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878796.282148:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290099456/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878796.282150:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878796.282157:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282166:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878796.282172:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878796.282173:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878796.282176:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878796.282181:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282182:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878796.282190:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878796.282194:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878796.282195:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878796.282201:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878796.282207:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878796.282214:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290099200:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878796.282218:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878796.282222:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878796.282225:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878796.282228:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878796.282233:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282243:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878796.282245:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:0.0:1788878796.282251:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878796.282271:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878796.282277:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878796.282281:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878796.282282:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282284:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878796.282288:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282320:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304407564 00000100:00000001:0.0:1788878796.282326:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304408105 00000100:00000001:0.0:1788878796.282329:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304407564 00000100:00000001:0.0:1788878796.282332:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304408105 00000100:00000001:0.0:1788878796.282335:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878796.282338:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282340:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878796.282346:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878796/real 1788878796] req@ffff94e5045e4700 x1875765290099200/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878796.282358:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878796.282360:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878796.282364:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878796.282367:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878796.282368:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282371:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878796.282373:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878796.282378:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290099200/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878796.282387:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282391:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878796.282394:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282396:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878796.282397:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282399:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878796.282414:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878796.282417:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878796.282421:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878796.282423:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878796.282427:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290099200:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878796.282432:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878796.282434:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878796.282436:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878796.282439:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878796.282441:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878796.282444:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878796.282447:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290099328:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878796.282451:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878796.282454:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878796.282456:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878796.282458:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878796.282462:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282469:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878796.282470:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:0.0:1788878796.282475:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878796.282491:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878796.282497:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878796.282499:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878796.282501:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282504:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878796.282507:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282511:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304407562 00000100:00000001:0.0:1788878796.282514:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304408105 00000100:00000001:0.0:1788878796.282516:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304407562 00000100:00000001:0.0:1788878796.282519:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304408105 00000100:00000001:0.0:1788878796.282522:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878796.282524:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282526:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878796.282531:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878796/real 1788878796] req@ffff94e5045e6680 x1875765290099328/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878796.282540:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878796.282541:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878796.282544:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878796.282545:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878796.282547:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282549:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878796.282551:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878796.282555:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290099328/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878796.282563:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282566:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878796.282569:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282571:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878796.282572:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282574:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878796.282586:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878796.282589:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878796.282592:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878796.282594:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878796.282599:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290099328:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878796.282603:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282606:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878796.282608:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878796.282610:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878796.282612:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878796.282613:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878796.282615:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878796.282618:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878796.282620:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878796.282622:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878796.282624:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282626:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878796.282628:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878796.282629:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878796.282632:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878796.282633:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878796.282635:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878796.282637:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878796.282638:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878796.282640:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878796.282642:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282644:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282648:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878796.282649:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282652:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878796.282653:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878796.282655:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878796.282656:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878796.282658:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878796.282661:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878796.282663:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878796.282665:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878796.282669:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290099456:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878796.282672:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878796.282674:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878796.282676:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878796.282677:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878796.282681:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282686:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878796.282688:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:0.0:1788878796.282692:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878796.282706:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878796.282713:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878796.282717:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878796.282719:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282723:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878796.282728:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282735:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304407562 00000100:00000001:0.0:1788878796.282739:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304408105 00000100:00000001:0.0:1788878796.282743:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304407562 00000100:00000001:0.0:1788878796.282747:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304408105 00000100:00000001:0.0:1788878796.282752:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878796.282756:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282759:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878796.282767:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878796/real 1788878796] req@ffff94e5045e6a00 x1875765290099456/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878796.282784:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878796.282787:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878796.282792:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878796.282795:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878796.282798:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282802:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878796.282805:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878796.282813:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290099456/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878796.282822:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282825:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878796.282828:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282830:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878796.282833:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282835:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878796.282847:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878796.282851:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878796.282855:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878796.282858:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878796.282862:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290099456:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878796.282866:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282868:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878796.282870:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878796.282871:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878796.282874:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878796.282876:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878796.282878:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878796.282880:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878796.282882:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878796.282884:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878796.282886:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282888:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.282892:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878796.282894:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.282896:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878796.282898:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878796.345018:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878796.345025:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878796.345028:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878796.345034:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878796.345037:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878796.345041:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.345043:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878796.345046:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878796.345049:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878797.049000:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878797.049005:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878797.049009:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878797.049011:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878797.049029:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878797.049031:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878797.049034:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878797.369020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878797.369028:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878797.369031:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878797.369037:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878797.369041:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878797.369045:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878797.369048:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878797.369052:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878797.369055:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878798.073000:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878798.073006:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878798.073010:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878798.073012:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878798.073071:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878798.073077:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878798.073082:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878798.393169:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878798.393178:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878798.393181:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878798.393186:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878798.393190:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878798.393195:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878798.393197:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878798.393199:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878798.393201:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878799.096996:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878799.097003:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878799.097009:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878799.097010:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878799.097032:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878799.097035:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878799.097037:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878799.417020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878799.417040:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878799.417043:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878799.417051:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878799.417055:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878799.417060:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878799.417064:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878799.417068:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878799.417071:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878800.121080:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878800.121095:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878800.121101:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878800.121103:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878800.121148:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878800.121151:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878800.121155:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878800.441035:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878800.441042:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878800.441044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878800.441048:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878800.441051:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878800.441054:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878800.441056:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878800.441058:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878800.441060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878801.144972:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878801.144977:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878801.144980:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878801.144983:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878801.144985:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878801.144987:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878801.144989:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878801.144991:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878801.144992:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878801.144994:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878801.145009:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878801.145011:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878801.145013:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878801.401092:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878801.401104:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878801.401109:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878801.401114:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878801.401120:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878801.401180:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9740 00000100:00080000:2.0:1788878801.401196:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878801.401198:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878801.401199:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878801.401205:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878801.401208:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878801.401210:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878801.401212:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878801.401213:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878801.401214:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878801.401218:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878801.401219:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878801.401221:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878801.401224:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878801.401226:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401228:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878801.401230:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878801.401237:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878801.401238:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401241:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878801.401243:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878801.401246:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401269:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878801.401277:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290099584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878801.401284:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878801.401288:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401329:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878801.401333:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878801.401337:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878801.401339:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878801.401342:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878801.401346:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878801.401349:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9740 00000100:00080000:2.0:1788878801.401354:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878801.401357:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878801.401359:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878801.401363:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878801.401365:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878801.401368:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878801.401371:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878801.401374:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878801.401377:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878801.401382:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878801.401385:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878801.401389:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878801.401398:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878801.401402:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401404:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878801.401409:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878801.401413:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401415:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878801.401420:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878801.401422:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401424:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878801.401426:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878801.401428:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401434:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878801.401447:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290099712/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878801.401486:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878801.401495:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401506:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878801.401513:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878801.401523:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878801.401529:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878801.401534:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878801.401538:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878801.401542:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9740 00000100:00080000:2.0:1788878801.401546:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878801.401548:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878801.401550:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878801.401555:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878801.401560:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878801.401565:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878801.401568:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878801.401571:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878801.401575:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878801.401579:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878801.401583:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878801.401585:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878801.401589:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878801.401591:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401594:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878801.401600:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878801.401621:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401624:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878801.401628:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878801.401629:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401630:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878801.401631:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878801.401633:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401637:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878801.401640:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290099840/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878801.401646:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878801.401647:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401649:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878801.401651:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878801.401656:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878801.401659:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878801.401661:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878801.401663:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878801.401665:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9740 00000100:00080000:2.0:1788878801.401669:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878801.401671:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878801.401673:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878801.401677:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878801.401680:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878801.401685:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878801.401686:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878801.401688:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878801.401692:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878801.401698:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878801.401703:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878801.401706:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878801.401712:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878801.401717:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401723:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878801.401728:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878801.401735:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401740:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878801.401747:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878801.401751:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401756:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878801.401759:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878801.401763:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401774:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878801.401785:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290099968/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878801.401807:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878801.401813:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878801.401821:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878801.465068:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878801.465074:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878801.465076:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878801.465079:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878801.465082:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878801.465085:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878801.465088:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878801.465108:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878801.465111:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878802.168978:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878802.168983:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878802.168986:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878802.168987:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878802.169048:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878802.169083:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878802.169085:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878802.489050:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878802.489058:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878802.489061:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878802.489067:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878802.489071:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878802.489075:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878802.489079:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878802.489082:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878802.489084:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878803.193062:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878803.193068:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878803.193072:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878803.193073:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878803.193111:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878803.193114:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878803.193118:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878803.513037:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878803.513043:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878803.513045:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878803.513049:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878803.513052:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878803.513056:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878803.513059:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878803.513078:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878803.513083:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878804.217110:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878804.217123:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878804.217127:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878804.217133:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878804.217141:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878804.217144:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878804.217148:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878804.217151:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878804.217156:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878804.217158:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878804.217213:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878804.217218:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878804.217224:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878804.537031:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878804.537053:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878804.537056:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878804.537063:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878804.537069:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878804.537073:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878804.537077:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878804.537081:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878804.537084:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878805.241101:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878805.241120:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878805.241128:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878805.241130:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878805.241158:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878805.241162:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878805.241165:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878805.561015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878805.561027:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878805.561031:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878805.561041:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878805.561048:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878805.561054:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878805.561059:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878805.561064:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878805.561069:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878806.264966:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878806.264972:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878806.264976:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878806.264977:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878806.265024:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878806.265027:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878806.265030:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878806.520985:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878806.520995:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878806.520997:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878806.521001:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878806.521006:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878806.521014:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9745 00000100:00080000:3.0:1788878806.521023:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878806.521027:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878806.521029:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878806.521036:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878806.521038:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878806.521041:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878806.521043:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878806.521044:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878806.521045:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878806.521049:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878806.521051:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878806.521053:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878806.521055:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878806.521058:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521060:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878806.521061:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878806.521071:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878806.521072:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521074:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878806.521076:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878806.521079:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521088:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878806.521095:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290100096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878806.521103:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878806.521107:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521118:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878806.521121:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878806.521124:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878806.521125:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878806.521126:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878806.521128:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878806.521130:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9745 00000100:00080000:3.0:1788878806.521133:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878806.521134:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878806.521135:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878806.521137:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878806.521138:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878806.521140:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878806.521141:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878806.521143:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878806.521144:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878806.521146:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878806.521149:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878806.521150:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878806.521155:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878806.521156:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521157:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878806.521161:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878806.521164:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521166:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878806.521172:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878806.521175:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521177:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878806.521180:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878806.521183:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521190:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878806.521196:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290100224/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878806.521208:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878806.521212:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521214:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878806.521231:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878806.521239:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878806.521242:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878806.521246:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878806.521249:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878806.521254:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9745 00000100:00080000:3.0:1788878806.521259:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878806.521263:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878806.521264:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878806.521266:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878806.521268:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878806.521271:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878806.521274:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878806.521275:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878806.521276:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878806.521279:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878806.521280:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878806.521282:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878806.521284:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878806.521286:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521287:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878806.521291:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878806.521294:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521297:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878806.521303:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878806.521306:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521307:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878806.521308:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878806.521310:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521315:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878806.521318:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290100352/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878806.521323:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878806.521325:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521327:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878806.521329:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878806.521332:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878806.521333:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878806.521334:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878806.521336:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878806.521338:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9745 00000100:00080000:3.0:1788878806.521340:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878806.521341:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878806.521342:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878806.521344:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878806.521344:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878806.521346:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878806.521347:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878806.521348:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878806.521349:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878806.521351:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878806.521353:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878806.521354:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878806.521356:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878806.521357:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521360:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878806.521362:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878806.521366:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521367:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878806.521370:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878806.521372:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521373:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878806.521374:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878806.521375:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521378:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878806.521384:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290100480/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878806.521404:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878806.521408:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878806.521410:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878806.584998:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878806.585003:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878806.585005:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878806.585008:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878806.585011:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878806.585013:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878806.585015:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878806.585017:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878806.585019:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878807.289016:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878807.289024:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878807.289031:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878807.289033:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878807.289057:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878807.289060:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878807.289064:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878807.609019:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878807.609028:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878807.609031:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878807.609037:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878807.609042:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878807.609047:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878807.609050:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878807.609053:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878807.609056:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878808.312997:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878808.313005:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878808.313012:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878808.313014:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878808.313039:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878808.313042:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878808.313046:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878808.633025:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878808.633034:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878808.633037:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878808.633043:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878808.633048:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878808.633052:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878808.633056:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878808.633059:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878808.633063:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878809.336977:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878809.336982:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878809.336988:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878809.336989:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878809.337008:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878809.337011:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878809.337014:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878809.656988:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878809.656997:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878809.656999:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878809.657006:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878809.657011:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878809.657015:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878809.657018:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878809.657021:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878809.657025:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878810.361053:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878810.361064:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878810.361071:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878810.361074:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878810.361170:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878810.361177:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878810.361183:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878810.680982:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878810.680990:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878810.680992:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878810.680997:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878810.681000:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878810.681004:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878810.681007:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878810.681009:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878810.681012:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878811.385178:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878811.385228:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878811.385240:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878811.385246:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878811.385251:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878811.385257:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878811.385263:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878811.385267:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878811.385272:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878811.385276:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878811.385351:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878811.385357:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878811.385363:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878811.641083:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878811.641094:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878811.641097:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878811.641101:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878811.641107:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878811.641127:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9750 00000100:00080000:0.0:1788878811.641140:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878811.641143:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878811.641145:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878811.641154:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878811.641157:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878811.641161:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878811.641164:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878811.641167:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878811.641169:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878811.641173:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878811.641176:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878811.641179:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878811.641206:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878811.641212:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641216:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878811.641220:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878811.641234:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878811.641237:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641243:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878811.641247:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878811.641254:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641266:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878811.641279:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290100608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878811.641291:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878811.641296:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641315:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878811.641320:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878811.641324:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878811.641325:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878811.641327:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878811.641331:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878811.641333:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9750 00000100:00080000:0.0:1788878811.641338:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878811.641341:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878811.641343:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878811.641347:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878811.641350:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878811.641353:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788878811.641356:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878811.641358:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878811.641360:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878811.641364:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878811.641367:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878811.641370:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878811.641380:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878811.641387:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641394:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878811.641400:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878811.641405:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641413:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878811.641447:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878811.641451:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641455:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878811.641460:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878811.641463:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641469:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878811.641475:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290100736/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878811.641514:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878811.641518:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641524:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878811.641531:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878811.641540:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878811.641545:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878811.641549:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878811.641554:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878811.641578:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9750 00000100:00080000:0.0:1788878811.641585:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878811.641592:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878811.641597:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878811.641605:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878811.641607:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878811.641609:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878811.641614:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878811.641620:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878811.641625:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878811.641630:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878811.641632:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878811.641634:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878811.641638:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878811.641641:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641643:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878811.641647:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878811.641649:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641651:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878811.641657:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878811.641659:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641663:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878811.641665:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878811.641672:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641682:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878811.641694:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290100864/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878811.641721:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878811.641724:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641728:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878811.641730:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878811.641733:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878811.641735:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878811.641737:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878811.641740:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878811.641742:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9750 00000100:00080000:0.0:1788878811.641748:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878811.641751:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878811.641753:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878811.641756:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878811.641757:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878811.641759:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788878811.641764:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878811.641772:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878811.641774:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878811.641780:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878811.641785:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878811.641791:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878811.641795:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878811.641804:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641814:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878811.641820:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878811.641824:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641827:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878811.641832:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878811.641836:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641839:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878811.641841:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878811.641845:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641853:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878811.641859:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290100992/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878811.641869:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878811.641874:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.641878:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878811.705010:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878811.705017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878811.705019:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878811.705024:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878811.705029:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878811.705033:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878811.705036:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878811.705039:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878811.705042:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878812.409071:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878812.409097:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878812.409102:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878812.409103:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878812.409123:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878812.409125:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878812.409128:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878812.729077:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878812.729091:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878812.729095:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878812.729103:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878812.729108:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878812.729113:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878812.729116:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878812.729119:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878812.729123:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878813.433001:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878813.433006:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878813.433009:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878813.433010:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878813.433028:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878813.433030:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878813.433032:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878813.753073:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878813.753084:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878813.753088:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878813.753095:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878813.753101:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878813.753106:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878813.753111:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878813.753115:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878813.753119:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878814.456974:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878814.456980:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878814.456982:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878814.456985:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878814.456989:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878814.456991:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878814.456992:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878814.456994:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878814.456996:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878814.456997:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878814.457057:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878814.457060:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878814.457063:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878814.776975:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878814.776981:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878814.776982:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878814.776986:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878814.776989:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878814.776992:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878814.776994:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878814.776996:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878814.776998:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878815.481012:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878815.481018:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878815.481022:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878815.481024:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878815.481045:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878815.481047:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878815.481049:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878815.801027:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878815.801035:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878815.801037:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878815.801043:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878815.801048:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878815.801052:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878815.801055:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878815.801059:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878815.801062:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878816.505040:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878816.505046:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878816.505051:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878816.505052:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878816.505072:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878816.505074:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878816.505077:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878816.761127:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878816.761169:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878816.761173:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878816.761177:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878816.761182:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878816.761190:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9755 00000100:00080000:3.0:1788878816.761202:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878816.761206:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878816.761209:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878816.761218:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878816.761222:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878816.761226:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878816.761230:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878816.761232:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878816.761234:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878816.761240:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878816.761244:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878816.761248:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878816.761252:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878816.761256:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761259:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878816.761262:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878816.761272:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878816.761274:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761277:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878816.761279:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878816.761283:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761294:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878816.761302:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290101120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878816.761312:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878816.761315:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761331:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878816.761334:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878816.761337:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878816.761339:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878816.761341:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878816.761343:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878816.761345:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9755 00000100:00080000:3.0:1788878816.761349:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878816.761351:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878816.761352:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878816.761356:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878816.761357:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878816.761360:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878816.761362:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878816.761364:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878816.761365:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878816.761368:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878816.761372:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878816.761377:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878816.761383:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878816.761389:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761393:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878816.761398:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878816.761401:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761403:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878816.761407:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878816.761410:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761413:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878816.761417:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878816.761421:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761447:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878816.761465:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290101248/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878816.761490:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878816.761494:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761504:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878816.761508:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878816.761512:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878816.761514:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878816.761517:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878816.761522:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878816.761529:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9755 00000100:00080000:3.0:1788878816.761538:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878816.761543:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878816.761545:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878816.761550:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878816.761551:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878816.761554:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878816.761571:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878816.761574:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878816.761581:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878816.761603:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878816.761605:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878816.761607:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878816.761610:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878816.761613:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761615:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878816.761620:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878816.761627:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761630:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878816.761639:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878816.761642:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761645:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878816.761649:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878816.761652:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761661:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878816.761676:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290101376/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878816.761718:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878816.761726:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761734:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878816.761744:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878816.761755:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878816.761760:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878816.761769:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878816.761781:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878816.761791:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9755 00000100:00080000:3.0:1788878816.761806:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878816.761815:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878816.761822:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878816.761849:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878816.761852:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878816.761855:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878816.761857:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878816.761860:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878816.761862:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878816.761869:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878816.761873:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878816.761877:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878816.761880:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878816.761882:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761885:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878816.761887:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878816.761891:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761897:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878816.761928:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878816.761935:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.761947:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878816.761954:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878816.761960:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.762032:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878816.762051:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290101504/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878816.762073:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878816.762078:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878816.762086:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878816.825044:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878816.825054:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878816.825058:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878816.825068:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878816.825075:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878816.825081:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878816.825085:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878816.825090:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878816.825094:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878817.529039:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878817.529050:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878817.529061:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878817.529064:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878817.529095:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878817.529099:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878817.529102:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878817.849035:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878817.849061:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878817.849063:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878817.849068:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878817.849072:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878817.849076:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878817.849079:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878817.849082:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878817.849084:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878818.553019:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878818.553027:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878818.553033:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878818.553035:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878818.553188:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878818.553194:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878818.553198:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878818.873046:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878818.873053:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878818.873055:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878818.873061:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878818.873064:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878818.873068:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878818.873070:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878818.873073:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878818.873075:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878819.577032:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878819.577040:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878819.577048:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878819.577050:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878819.577078:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878819.577083:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878819.577088:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878819.833606:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878819.833614:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878819.833623:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878819.833651:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878816/real 1788878819] req@ffff94e50449c700 x1875765290101248/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878819.833683:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878819.833687:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878819.833698:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878819.833704:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878819.833707:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.833712:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878819.833717:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878819.833727:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290101248/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878819.833746:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.833753:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878819.833781:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.833785:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878819.833788:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.833793:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878819.833817:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878819.833824:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878819.833834:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878819.833840:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878819.833849:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290101248:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878819.833859:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878819.833867:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878816/real 1788878819] req@ffff94e50449f480 x1875765290101376/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878819.833886:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878819.833889:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878819.833896:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878819.833899:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878819.833972:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.833977:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878819.833981:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878819.833991:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290101376/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878819.834006:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834010:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878819.834016:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834020:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878819.834024:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834028:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878819.834070:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878819.834078:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878819.834087:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878819.834093:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878819.834099:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290101376:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878819.834107:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878819.834114:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878816/real 1788878819] req@ffff94e50449f800 x1875765290101120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878819.834158:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878819.834162:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878819.834169:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878819.834173:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878819.834176:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834180:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878819.834183:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878819.834192:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290101120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878819.834206:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834210:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878819.834214:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834218:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878819.834220:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834223:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878819.834227:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878819.834231:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878819.834238:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878819.834242:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878819.834249:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290101120:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878819.834257:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878819.834260:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878819.834263:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878819.834268:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878819.834276:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878819.834280:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878819.834284:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878819.834290:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878819.834294:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878819.834298:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878819.834301:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834304:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878819.834306:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878819.834309:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878819.834312:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878819.834314:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878819.834316:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878819.834320:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878819.834322:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878819.834325:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878819.834327:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834330:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878819.834332:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878819.834334:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878819.834337:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878819.834340:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878819.834342:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878819.834345:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878819.834347:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878819.834350:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878819.834353:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834357:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878819.834361:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878819.834363:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878819.834365:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878819.834373:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878816/real 1788878819] req@ffff94e50449d180 x1875765290101504/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878819.834391:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878819.834393:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878819.834398:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878819.834401:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878819.834404:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834407:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878819.834409:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878819.834416:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290101504/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878819.834427:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834431:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878819.834435:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834437:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878819.834439:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834443:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878819.834464:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878819.834469:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878819.834475:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878819.834479:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878819.834485:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290101504:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878819.834492:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834496:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878819.834498:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878819.834501:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878819.834505:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878819.834508:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878819.834510:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878819.834514:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878819.834517:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878819.834521:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878819.834524:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834527:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.834538:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878819.834543:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878819.834548:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878819.834552:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878819.897025:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878819.897051:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878819.897056:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878819.897065:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878819.897071:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878819.897077:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.897082:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878819.897087:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878819.897092:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878820.601015:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878820.601023:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878820.601031:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878820.601033:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878820.601060:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878820.601063:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878820.601068:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878820.921022:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878820.921032:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878820.921035:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878820.921041:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878820.921046:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878820.921051:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878820.921055:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878820.921058:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878820.921062:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878821.625015:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878821.625022:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878821.625028:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878821.625031:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878821.625034:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878821.625038:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878821.625041:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878821.625043:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878821.625045:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878821.625058:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878821.625078:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878821.625080:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878821.625082:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878821.880999:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878821.881009:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878821.881012:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878821.881016:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878821.881021:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878821.881029:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9760 00000100:00080000:2.0:1788878821.881040:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878821.881044:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878821.881047:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878821.881054:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878821.881058:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878821.881062:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878821.881065:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878821.881067:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878821.881070:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878821.881075:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878821.881079:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878821.881082:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878821.881085:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878821.881088:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878821.881090:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878821.881093:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878821.881102:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878821.881104:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878821.881107:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878821.881110:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878821.881130:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878821.881142:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878821.881151:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290101632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878821.881162:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878821.881166:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878821.881179:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878821.881183:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878821.881186:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878821.881188:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878821.881189:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878821.881193:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878821.881196:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9760 00000100:00080000:2.0:1788878821.881201:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878821.881203:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878821.881205:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878821.881209:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878821.881212:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788878821.881212:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878821.881217:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878821.881220:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788878821.881221:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878821.881224:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878821.881226:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878821.881227:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878821.881232:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878821.881232:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:0.0:1788878821.881238:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1788878821.881240:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:0.0:1788878821.881242:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878821.881248:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878821.881249:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878821.881252:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:0.0:1788878821.881259:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290101632:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878821.881263:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878821.881266:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878821.881268:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878821.881272:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788878821.881272:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:0.0:1788878821.881278:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788878821.881279:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788878821.881283:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:2.0:1788878821.881284:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878821.881288:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878821.881290:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878821.881293:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878821.881296:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878821.881298:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878821.881300:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878821.881302:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881309:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878821.881310:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881314:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000100:00100000:2.0:1788878821.881320:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290101760/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878821.881322:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878821.881332:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878821.881335:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878821.881338:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878821.881340:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878821.881344:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878821.881346:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878821.881348:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878821.881352:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878821.881354:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9760 00000100:00080000:2.0:1788878821.881359:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878821.881361:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878821.881363:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:0.0:1788878821.881367:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:2.0:1788878821.881368:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878821.881372:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878821.881374:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878821.881377:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878821.881380:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878821.881382:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878821.881382:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878821.881389:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788878821.881390:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878821.881394:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878821.881396:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881399:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:2.0:1788878821.881402:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878821.881406:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878821.881411:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878821.881413:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304433705 02000000:00000001:2.0:1788878821.881417:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878821.881421:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878821.881426:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878821.881427:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00000001:2.0:1788878821.881430:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878821.881434:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881437:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304433705 02000000:00000001:2.0:1788878821.881439:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878821.881442:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878821.881445:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878821.881447:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878821.881449:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1788878821.881450:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878821.881458:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878821.881463:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290101888/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878821.881465:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878821.881472:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881478:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878821.881483:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878821.881488:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878821.881489:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878821/real 1788878821] req@ffff94e5045e7800 x1875765290101632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878821.881494:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878821.881500:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878821.881505:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878821.881507:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878821.881510:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878821.881515:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000400:00000001:0.0:1788878821.881517:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1788878821.881520:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9760 00000400:00000001:0.0:1788878821.881522:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1788878821.881527:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878821.881530:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878821.881531:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878821.881534:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878821.881536:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000020:00000001:2.0:1788878821.881538:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878821.881540:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788878821.881541:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878821.881546:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878821.881547:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1788878821.881550:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878821.881552:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878821.881554:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878821.881557:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:0.0:1788878821.881561:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290101632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878821.881563:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878821.881570:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878821.881575:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878821.881579:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878821.881581:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878821.881583:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881588:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878821.881588:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878821.881592:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878821.881595:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1788878821.881597:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881600:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878821.881601:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881605:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878821.881607:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878821.881609:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1788878821.881611:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881614:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878821.881615:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878821.881618:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00080000:0.0:1788878821.881622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1788878821.881622:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878821.881631:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878821.881632:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878821.881637:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878821.881640:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290102016/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878821.881645:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290101632:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878821.881658:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878821.881660:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878821.881663:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878821.881664:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881667:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788878821.881669:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878821.881672:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878821.881678:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878821.881681:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878821.881683:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878821.881689:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878821.881691:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878821.881694:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878821.881695:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.881698:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.881705:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878821.881707:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881709:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878821.881711:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878821.881712:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878821.881714:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878821.881715:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878821.881718:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878821.881720:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878821.881722:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878821.881726:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290101760:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878821.881730:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878821.881732:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878821.881733:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878821.881735:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878821.881738:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881745:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878821.881747:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 00000100:00000001:0.0:1788878821.881751:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878821.881766:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878821.881771:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878821.881774:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878821.881775:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881777:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878821.881779:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881783:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304433163 00000100:00000001:0.0:1788878821.881786:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304433705 00000100:00000001:0.0:1788878821.881788:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304433163 00000100:00000001:0.0:1788878821.881791:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304433705 00000100:00000001:0.0:1788878821.881793:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878821.881795:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881797:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878821.881801:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878821/real 1788878821] req@ffff94e5045e6300 x1875765290101760/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878821.881809:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878821.881810:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878821.881812:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878821.881814:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878821.881815:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.881817:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878821.881818:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878821.881822:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290101760/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878821.881829:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.881831:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878821.881834:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.881835:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878821.881837:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.881839:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878821.881853:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878821.881856:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878821.881859:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878821.881861:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878821.881865:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290101760:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878821.881869:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878821.881871:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878821.881873:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878821.881875:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878821.881877:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878821.881879:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878821.881882:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290101888:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878821.881885:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878821.881887:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878821.881889:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878821.881890:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878821.881894:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881899:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878821.881924:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:0.0:1788878821.881929:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878821.881945:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878821.881950:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878821.881952:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878821.881954:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881956:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878821.881959:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881968:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304433162 00000100:00000001:0.0:1788878821.881970:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304433705 00000100:00000001:0.0:1788878821.881973:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304433162 00000100:00000001:0.0:1788878821.881975:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304433705 00000100:00000001:0.0:1788878821.881977:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878821.881980:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.881982:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878821.881987:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878821/real 1788878821] req@ffff94e5045e5500 x1875765290101888/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878821.881996:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878821.881997:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878821.882001:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878821.882003:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878821.882004:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.882006:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878821.882007:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878821.882011:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290101888/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878821.882023:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.882025:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878821.882028:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.882030:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878821.882031:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.882032:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878821.882044:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878821.882047:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878821.882050:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878821.882052:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878821.882055:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290101888:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878821.882058:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878821.882060:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878821.882061:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878821.882063:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878821.882065:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878821.882066:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878821.882069:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290102016:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878821.882072:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878821.882073:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878821.882074:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878821.882076:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878821.882078:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.882083:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878821.882084:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:0.0:1788878821.882087:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878821.882099:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878821.882103:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878821.882106:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878821.882107:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.882109:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878821.882111:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.882131:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304433162 00000100:00000001:0.0:1788878821.882133:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304433705 00000100:00000001:0.0:1788878821.882136:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304433162 00000100:00000001:0.0:1788878821.882138:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304433705 00000100:00000001:0.0:1788878821.882139:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878821.882142:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.882143:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878821.882147:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878821/real 1788878821] req@ffff94e5045e5c00 x1875765290102016/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878821.882155:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878821.882156:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878821.882159:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878821.882161:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878821.882162:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.882164:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878821.882165:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878821.882169:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290102016/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878821.882175:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.882177:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878821.882179:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.882180:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878821.882181:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.882183:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878821.882195:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878821.882198:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878821.882201:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878821.882203:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878821.882207:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290102016:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878821.882211:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.882213:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878821.882214:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878821.882215:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878821.882218:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878821.882219:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878821.882221:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878821.882224:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878821.882225:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878821.882227:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878821.882229:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.882230:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878821.882231:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878821.882233:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878821.882235:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878821.882236:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878821.882237:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878821.882239:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878821.882240:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878821.882241:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878821.882242:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.882244:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878821.882245:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878821.882246:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878821.882248:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878821.882249:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878821.882250:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878821.882252:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878821.882253:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878821.882255:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878821.882256:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.882258:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.882262:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878821.882264:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.882266:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878821.882267:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878821.944964:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878821.944969:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878821.944970:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878821.944974:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878821.944977:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878821.944980:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.944981:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878821.944983:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878821.944985:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878822.648982:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878822.648988:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878822.648992:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878822.648993:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878822.649071:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878822.649075:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878822.649079:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878822.969035:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878822.969042:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878822.969044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878822.969050:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878822.969055:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878822.969060:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878822.969063:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878822.969066:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878822.969070:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878823.673043:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878823.673050:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878823.673057:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878823.673059:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878823.673085:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878823.673088:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878823.673092:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878823.993059:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878823.993084:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878823.993088:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878823.993096:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878823.993129:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878823.993135:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878823.993140:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878823.993144:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878823.993149:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878824.697137:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878824.697152:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878824.697156:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878824.697163:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878824.697171:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878824.697175:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878824.697178:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878824.697182:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878824.697188:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878824.697191:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878824.697231:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878824.697235:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878824.697240:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878825.017103:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878825.017117:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878825.017119:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878825.017126:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878825.017130:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878825.017135:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878825.017137:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878825.017140:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878825.017154:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878825.721113:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878825.721126:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878825.721149:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878825.721151:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878825.721198:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878825.721203:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878825.721207:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878826.041228:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878826.041240:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878826.041244:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878826.041252:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878826.041259:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878826.041264:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878826.041278:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878826.041283:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878826.041287:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878826.745048:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878826.745056:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878826.745062:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878826.745065:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878826.745256:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878826.745261:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878826.745266:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878827.001054:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878827.001111:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878827.001116:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878827.001138:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878827.001144:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878827.001154:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9765 00000100:00080000:2.0:1788878827.001169:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878827.001174:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878827.001177:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878827.001188:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878827.001191:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878827.001196:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878827.001201:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878827.001205:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878827.001207:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878827.001214:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878827.001217:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878827.001220:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878827.001224:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878827.001228:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878827.001230:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878827.001233:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878827.001245:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878827.001247:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878827.001250:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878827.001253:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878827.001259:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878827.001271:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878827.001281:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5f80 x1875765290102144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878827.001293:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878827.001299:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878827.001325:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878827.001349:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878827.001355:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878827.001358:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878827.001362:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878827.001368:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878827.001374:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9765 00000100:00080000:2.0:1788878827.001382:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878827.001387:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878827.001391:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878827.001399:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878827.001403:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878827.001407:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878827.001413:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878827.001418:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878827.001420:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878827.001426:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788878827.001427:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878827.001436:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878827.001437:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.001443:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788878827.001444:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878827.001451:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788878827.001456:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878827.001457:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878827.001469:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878827.001470:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788878827.001474:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:0.0:1788878827.001477:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:2.0:1788878827.001481:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788878827.001484:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788878827.001493:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878827.001501:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290102144:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878827.001502:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.001515:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878827.001516:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878827.001522:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878827.001523:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878827.001530:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878827.001531:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878827.001538:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878827.001541:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878827.001547:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878827.001551:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878827.001568:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.001572:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878827.001577:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074556288 : -117763634995328 : ffff94e5045e5f80) 00000100:00100000:2.0:1788878827.001585:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4380 x1875765290102272/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878827.001586:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878827.001609:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878827.001614:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878827.001623:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878827.001627:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1788878827.001637:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788878827.001639:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878827.001646:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878827.001648:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878827.001654:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878827.001658:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788878827.001661:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9765 00000100:00000001:0.0:1788878827.001667:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788878827.001671:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878827.001673:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878827.001683:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878827.001684:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878827.001692:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878827.001695:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878827.001701:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878827.001719:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304438825 00000020:00000001:2.0:1788878827.001721:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878827.001726:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878827.001729:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878827.001732:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878827.001734:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878827.001735:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878827.001745:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878827.001748:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.001751:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304438825 02000000:00000001:2.0:1788878827.001751:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878827.001757:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878827.001761:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.001764:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878827.001765:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878827.001770:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878827.001774:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878827.001776:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.001786:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878827.001790:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878827.001795:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878827.001795:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878827.001804:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878827.001806:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1788878827.001807:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878827.001813:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878827.001821:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878827.001823:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878827/real 1788878827] req@ffff94e5045e5f80 x1875765290102144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878827.001832:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290102400/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1788878827.001857:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788878827.001857:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:0.0:1788878827.001864:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878827.001865:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878827.001874:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.001875:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878827.001882:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00080000:2.0:1788878827.001883:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878827.001887:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878827.001891:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878827.001893:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878827.001895:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878827.001898:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878827.001899:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878827.001905:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00100000:0.0:1788878827.001909:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5f80 x1875765290102144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878827.001912:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9765 00000100:00080000:2.0:1788878827.001919:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878827.001920:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878827.001922:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878827.001926:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878827.001928:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878827.001931:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878827.001964:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.001971:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1788878827.001979:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878827.001980:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878827.001988:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878827.001991:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878827.001997:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878827.002021:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878827.002025:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002028:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878827.002035:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878827.002038:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1788878827.002046:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878827.002047:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788878827.002061:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878827.002068:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788878827.002070:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878827.002099:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878827.002121:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878827.002124:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878827.002129:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002130:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878827.002140:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290102144:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878827.002144:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878827.002146:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878827.002150:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002152:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878827.002155:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788878827.002159:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878827.002160:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878827.002167:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878827.002169:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878827.002176:0:2361:0:(genops.c:1216:class_import_put()) Process entered 02000000:00000001:2.0:1788878827.002176:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878827.002181:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878827.002186:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878827.002188:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878827.002192:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878827.002192:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002196:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002200:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878827.002206:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290102528/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878827.002209:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878827.002213:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002220:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878827.002224:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878827.002227:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878827.002229:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878827.002234:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002235:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878827.002239:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878827.002245:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878827.002246:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878827.002251:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878827.002255:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878827.002261:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290102272:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878827.002264:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878827.002267:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878827.002269:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878827.002271:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878827.002275:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002283:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878827.002285:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074549120 : -117763635002496 : ffff94e5045e4380) 00000100:00000001:0.0:1788878827.002290:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878827.002311:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878827.002319:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878827.002323:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878827.002325:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002328:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878827.002332:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002339:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304438284 00000100:00000001:0.0:1788878827.002343:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304438825 00000100:00000001:0.0:1788878827.002347:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304438284 00000100:00000001:0.0:1788878827.002351:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304438825 00000100:00000001:0.0:1788878827.002355:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878827.002376:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002380:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878827.002389:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878827/real 1788878827] req@ffff94e5045e4380 x1875765290102272/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878827.002405:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878827.002407:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878827.002411:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878827.002414:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878827.002416:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002418:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878827.002420:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878827.002425:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4380 x1875765290102272/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878827.002434:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002437:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878827.002440:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002442:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878827.002444:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002446:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878827.002461:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878827.002464:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878827.002468:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878827.002470:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878827.002474:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290102272:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878827.002479:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878827.002481:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878827.002483:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878827.002486:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878827.002489:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878827.002491:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878827.002495:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290102400:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878827.002499:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878827.002501:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878827.002503:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878827.002505:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878827.002509:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002516:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878827.002517:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:0.0:1788878827.002522:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878827.002538:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878827.002544:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878827.002547:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878827.002549:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002551:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878827.002554:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002558:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304438282 00000100:00000001:0.0:1788878827.002561:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304438825 00000100:00000001:0.0:1788878827.002564:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304438282 00000100:00000001:0.0:1788878827.002567:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304438825 00000100:00000001:0.0:1788878827.002570:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878827.002572:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002574:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878827.002579:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878827/real 1788878827] req@ffff94e5045e5c00 x1875765290102400/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878827.002588:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878827.002589:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878827.002592:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878827.002594:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878827.002596:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002598:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878827.002600:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878827.002604:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290102400/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878827.002612:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002615:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878827.002617:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002619:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878827.002621:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002623:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878827.002634:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878827.002637:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878827.002641:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878827.002644:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878827.002648:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290102400:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878827.002652:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002654:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878827.002656:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878827.002658:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878827.002660:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878827.002662:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878827.002664:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878827.002666:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878827.002668:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878827.002670:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878827.002672:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002674:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878827.002675:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878827.002677:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878827.002679:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878827.002681:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878827.002682:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878827.002684:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878827.002685:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878827.002687:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878827.002689:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002691:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002695:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878827.002697:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002699:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878827.002701:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878827.002703:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878827.002704:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878827.002706:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878827.002708:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878827.002710:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878827.002746:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878827.002757:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290102528:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878827.002763:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878827.002766:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878827.002769:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878827.002772:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878827.002776:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002794:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878827.002796:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:0.0:1788878827.002802:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878827.002821:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878827.002826:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878827.002829:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878827.002831:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002833:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878827.002835:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002839:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304438282 00000100:00000001:0.0:1788878827.002842:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304438825 00000100:00000001:0.0:1788878827.002844:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304438282 00000100:00000001:0.0:1788878827.002846:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304438825 00000100:00000001:0.0:1788878827.002849:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878827.002850:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.002852:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878827.002856:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878827/real 1788878827] req@ffff94e5045e5500 x1875765290102528/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878827.002882:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878827.002884:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878827.002887:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878827.002889:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878827.002891:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002894:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878827.002895:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878827.002900:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290102528/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878827.002909:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002911:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878827.002915:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002948:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878827.002951:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.002955:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878827.002974:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878827.002980:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878827.002987:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878827.002991:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878827.002998:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290102528:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878827.003006:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.003010:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878827.003013:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878827.003016:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878827.003020:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878827.003023:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878827.003026:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878827.003030:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878827.003032:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878827.003036:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878827.003038:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.003043:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.003055:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878827.003057:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.003060:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878827.003062:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878827.065016:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878827.065025:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878827.065029:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878827.065038:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878827.065055:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878827.065061:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.065066:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878827.065070:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878827.065075:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878827.769021:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878827.769028:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878827.769034:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878827.769036:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878827.769097:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878827.769100:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878827.769110:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878828.793054:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878828.793060:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878828.793091:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878828.793093:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878828.793145:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878828.793149:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878828.793152:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878829.816997:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878829.817002:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878829.817006:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878829.817008:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878829.817025:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878829.817027:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878829.817029:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878830.841100:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878830.841121:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878830.841125:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878830.841126:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878830.841189:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878830.841193:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878830.841196:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878831.865004:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878831.865018:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878831.865024:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878831.865027:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878831.865030:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878831.865033:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878831.865036:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878831.865038:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878831.865040:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878831.865042:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878831.865079:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878831.865082:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878831.865085:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878832.121072:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878832.121087:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878832.121107:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878832.121112:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878832.121118:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878832.121128:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9770 00000100:00080000:2.0:1788878832.121143:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878832.121148:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878832.121151:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878832.121163:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878832.121167:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878832.121171:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878832.121174:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878832.121176:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878832.121178:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878832.121184:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878832.121187:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878832.121190:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878832.121193:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878832.121197:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878832.121199:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878832.121202:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878832.121216:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878832.121219:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878832.121224:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878832.121227:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878832.121233:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878832.121249:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878832.121263:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290102656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878832.121281:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878832.121288:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878832.121309:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878832.121312:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878832.121316:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878832.121318:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878832.121320:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878832.121323:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878832.121326:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9770 00000100:00080000:2.0:1788878832.121331:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878832.121335:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878832.121337:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878832.121341:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878832.121342:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878832.121345:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878832.121348:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878832.121351:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878832.121353:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878832.121357:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878832.121360:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878832.121363:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878832.121369:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878832.121371:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878832.121374:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878832.121376:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:2.0:1788878832.121379:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878832.121386:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.121391:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878832.121392:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.121406:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788878832.121407:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878832.121411:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.121414:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878832.121416:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878832.121420:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878832.121423:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878832.121424:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878832.121436:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878832.121439:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878832.121446:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1788878832.121449:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290102784/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788878832.121454:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878832.121472:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290102656:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878832.121476:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878832.121483:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.121486:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788878832.121494:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878832.121496:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878832.121503:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878832.121505:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1788878832.121511:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1788878832.121515:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878832.121521:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:0.0:1788878832.121523:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878832.121524:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878832.121535:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878832.121539:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9770 00000100:00080000:2.0:1788878832.121546:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878832.121549:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878832.121551:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878832.121556:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878832.121558:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1788878832.121558:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878832.121564:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878832.121565:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 00000020:00000001:2.0:1788878832.121571:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878832.121576:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878832.121578:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878832.121579:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788878832.121585:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878832.121588:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878832.121590:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878832.121594:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878832.121596:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878832.121598:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878832.121600:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878832.121603:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878832.121605:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878832.121610:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878832.121612:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878832.121613:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878832.121615:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878832.121618:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878832.121625:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878832.121631:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290102912/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878832.121639:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878832.121642:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878832.121646:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788878832.121650:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788878832.121651:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878832.121660:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878832.121663:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878832.121666:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878832.121670:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878832.121673:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9770 00000100:00000001:0.0:1788878832.121679:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788878832.121679:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878832.121687:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878832.121689:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878832.121692:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1788878832.121693:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878832.121698:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878832.121699:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878832.121701:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878832.121711:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:2.0:1788878832.121711:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878832.121717:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878832.121719:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878832.121724:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878832.121725:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878832.121738:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878832.121741:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878832.121742:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304443945 02000000:00000001:2.0:1788878832.121745:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878832.121750:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878832.121752:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878832.121755:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878832.121759:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.121760:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878832.121762:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878832.121772:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878832.121774:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878832.121776:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878832.121778:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878832.121779:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304443945 02000000:00000001:2.0:1788878832.121782:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878832.121790:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.121794:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00100000:2.0:1788878832.121797:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290103040/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878832.121804:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878832.121816:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878832.121818:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878832.121822:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.121826:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878832.121829:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878832.121844:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878832/real 1788878832] req@ffff94e5045e6300 x1875765290102656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878832.121863:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878832.121867:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878832.121873:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878832.121877:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878832.121880:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.121883:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878832.121886:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878832.121893:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290102656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878832.121948:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.121952:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878832.121956:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.121958:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878832.121959:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.121972:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878832.121975:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878832.121979:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878832.121984:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878832.121987:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878832.121993:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290102656:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878832.121999:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122002:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878832.122004:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878832.122007:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878832.122012:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878832.122016:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878832.122018:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878832.122025:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878832.122027:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878832.122031:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878832.122032:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122035:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122060:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878832.122063:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.122066:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878832.122068:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878832.122070:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878832.122072:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878832.122074:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878832.122077:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878832.122079:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878832.122081:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878832.122085:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290102784:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878832.122089:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878832.122091:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878832.122103:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878832.122105:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878832.122108:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.122115:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878832.122117:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000100:00000001:0.0:1788878832.122122:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878832.122146:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878832.122151:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878832.122154:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878832.122155:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.122157:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878832.122160:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.122165:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304443404 00000100:00000001:0.0:1788878832.122168:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304443945 00000100:00000001:0.0:1788878832.122171:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304443404 00000100:00000001:0.0:1788878832.122173:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304443945 00000100:00000001:0.0:1788878832.122176:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878832.122178:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.122180:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878832.122185:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878832/real 1788878832] req@ffff94e5045e7800 x1875765290102784/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878832.122195:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878832.122197:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878832.122200:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878832.122202:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878832.122204:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122206:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878832.122209:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878832.122213:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290102784/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878832.122221:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122224:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878832.122227:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122229:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878832.122231:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122233:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878832.122252:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878832.122256:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878832.122260:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878832.122264:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878832.122268:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290102784:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878832.122272:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878832.122274:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878832.122276:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878832.122279:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878832.122281:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878832.122283:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878832.122287:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290102912:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878832.122291:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878832.122294:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878832.122295:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878832.122298:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878832.122301:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.122309:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878832.122311:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:0.0:1788878832.122316:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878832.122331:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878832.122337:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878832.122341:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878832.122342:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.122345:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878832.122347:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.122352:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304443402 00000100:00000001:0.0:1788878832.122355:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304443945 00000100:00000001:0.0:1788878832.122357:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304443402 00000100:00000001:0.0:1788878832.122360:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304443945 00000100:00000001:0.0:1788878832.122362:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878832.122365:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.122367:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878832.122372:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878832/real 1788878832] req@ffff94e5045e7100 x1875765290102912/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878832.122382:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878832.122384:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878832.122387:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878832.122389:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878832.122391:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122393:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878832.122395:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878832.122400:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290102912/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878832.122408:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122411:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878832.122414:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122416:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878832.122418:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122420:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878832.122437:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878832.122441:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878832.122444:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878832.122447:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878832.122449:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290102912:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878832.122452:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878832.122454:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878832.122455:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878832.122457:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878832.122459:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878832.122460:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878832.122463:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290103040:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878832.122465:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878832.122467:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878832.122468:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878832.122470:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878832.122473:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.122478:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878832.122479:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:0.0:1788878832.122483:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878832.122498:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878832.122504:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878832.122507:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878832.122508:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.122511:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878832.122514:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.122519:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304443402 00000100:00000001:0.0:1788878832.122522:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304443945 00000100:00000001:0.0:1788878832.122525:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304443402 00000100:00000001:0.0:1788878832.122527:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304443945 00000100:00000001:0.0:1788878832.122530:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878832.122533:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.122535:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878832.122540:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878832/real 1788878832] req@ffff94e5045e4a80 x1875765290103040/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878832.122557:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878832.122559:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878832.122562:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878832.122564:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878832.122565:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122567:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878832.122569:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878832.122574:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290103040/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878832.122583:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122586:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878832.122589:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122591:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878832.122593:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122595:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878832.122614:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878832.122617:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878832.122621:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878832.122624:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878832.122628:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290103040:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878832.122633:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122635:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878832.122637:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878832.122639:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878832.122642:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878832.122644:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878832.122645:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878832.122648:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878832.122650:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878832.122653:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878832.122654:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122657:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878832.122659:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878832.122660:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878832.122663:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878832.122665:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878832.122666:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878832.122669:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878832.122670:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878832.122672:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878832.122673:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122675:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878832.122677:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878832.122678:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878832.122680:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878832.122681:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878832.122683:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878832.122685:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878832.122686:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878832.122688:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878832.122689:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122692:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878832.122696:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878832.122698:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878832.122700:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878832.122702:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878832.889191:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878832.889201:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878832.889208:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878832.889210:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878832.889243:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878832.889259:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878832.889263:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878833.913119:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878833.913126:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878833.913133:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878833.913136:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878833.913175:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878833.913177:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878833.913180:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878834.936983:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878834.936992:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878834.936994:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878834.936998:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878834.937002:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878834.937004:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878834.937007:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878834.937010:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878834.937013:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878834.937015:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878834.937056:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878834.937059:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878834.937062:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878835.961073:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878835.961080:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878835.961085:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878835.961086:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878835.961108:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878835.961111:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878835.961114:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878836.985006:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878836.985042:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878836.985049:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878836.985051:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878836.985078:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878836.985080:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878836.985082:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878837.241043:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878837.241074:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878837.241077:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878837.241082:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878837.241090:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878837.241101:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9776 00000100:00080000:2.0:1788878837.241117:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878837.241122:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878837.241125:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878837.241137:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878837.241140:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878837.241145:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878837.241148:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878837.241150:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878837.241153:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878837.241159:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878837.241162:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878837.241165:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878837.241169:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878837.241173:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878837.241175:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878837.241177:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878837.241191:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878837.241193:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878837.241197:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878837.241200:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878837.241205:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878837.241219:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878837.241229:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290103168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878837.241240:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878837.241245:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878837.241266:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878837.241270:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878837.241274:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878837.241277:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878837.241280:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878837.241283:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878837.241285:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9776 00000100:00080000:2.0:1788878837.241289:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878837.241291:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878837.241292:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878837.241297:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878837.241298:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878837.241300:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878837.241302:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878837.241304:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878837.241306:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878837.241310:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878837.241311:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878837.241318:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.241320:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788878837.241321:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878837.241335:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878837.241340:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878837.241343:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878837.241348:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878837.241349:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878837.241353:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878837.241355:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878837.241357:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878837.241363:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:0.0:1788878837.241370:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290103168:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1788878837.241371:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878837.241379:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1788878837.241381:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878837.241385:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878837.241388:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878837.241388:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878837.241398:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878837.241402:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878837.241405:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878837.241408:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878837.241409:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878837.241411:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878837.241413:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878837.241418:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878837.241426:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290103296/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878837.241433:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878837.241433:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878837.241440:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 00000100:00000001:2.0:1788878837.241441:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878837.241454:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.241455:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788878837.241463:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878837.241467:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878837.241469:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878837.241470:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878837.241474:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878837.241477:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9776 00000100:00080000:2.0:1788878837.241481:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878837.241482:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878837.241483:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878837.241487:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878837.241488:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878837.241490:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000800:00000100:0.0:1788878837.241510:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:2.0:1788878837.241510:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878837.241519:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878837.241522:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878837.241527:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788878837.241528:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878837.241533:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.241534:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788878837.241536:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878837.241538:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.241542:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:2.0:1788878837.241542:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878837.241549:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.241551:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878837.241552:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878837.241562:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878837.241563:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304449065 00000008:00000001:2.0:1788878837.241568:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.241581:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878837.241581:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878837.241591:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878837.241593:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304449065 02000000:00000001:2.0:1788878837.241594:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.241618:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878837.241620:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878837.241623:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878837.241623:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878837.241631:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878837.241631:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.241646:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878837.241650:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878837.241657:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878837/real 1788878837] req@ffff94e5045e5180 x1875765290103168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878837.241658:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290103424/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1788878837.241722:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788878837.241723:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878837.241730:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788878837.241731:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878837.241736:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.241741:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:2.0:1788878837.241741:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878837.241750:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878837.241752:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878837.241753:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878837.241757:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878837.241760:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878837.241761:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878837.241769:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9776 00000100:00000001:0.0:1788878837.241772:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1788878837.241775:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878837.241777:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878837.241778:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878837.241780:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878837.241790:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878837.241791:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878837.241797:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:0.0:1788878837.241798:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290103168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788878837.241807:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878837.241818:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878837.241828:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878837.241829:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.241852:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878837.241858:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878837.241860:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878837.241865:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878837.241866:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.241870:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878837.241871:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878837.241880:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1788878837.241881:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878837.241886:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788878837.241887:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878837.241898:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00080000:0.0:1788878837.241899:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1788878837.241901:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878837.241904:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878837.241905:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878837.241909:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878837.241911:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878837.241914:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878837.241916:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878837.241919:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878837.241923:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878837.241927:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290103552/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878837.241942:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00080000:0.0:1788878837.241946:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878837.241947:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.241957:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878837.241963:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290103168:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878837.241969:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.241971:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878837.241975:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878837.241978:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788878837.241980:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878837.241983:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878837.241986:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878837.241989:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878837.241996:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878837.241998:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878837.242002:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878837.242004:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242006:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242027:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878837.242029:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.242032:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878837.242035:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878837.242036:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878837.242038:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878837.242039:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878837.242042:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878837.242044:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878837.242047:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878837.242051:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290103296:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878837.242056:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878837.242059:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878837.242060:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878837.242062:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878837.242066:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.242074:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878837.242076:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:0.0:1788878837.242081:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878837.242099:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878837.242104:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878837.242108:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878837.242109:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.242110:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878837.242113:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.242117:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304448524 00000100:00000001:0.0:1788878837.242120:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304449065 00000100:00000001:0.0:1788878837.242121:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304448524 00000100:00000001:0.0:1788878837.242123:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304449065 00000100:00000001:0.0:1788878837.242125:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878837.242126:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.242128:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878837.242131:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878837/real 1788878837] req@ffff94e5045e5880 x1875765290103296/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878837.242138:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878837.242139:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878837.242141:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878837.242142:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878837.242144:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242145:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878837.242146:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878837.242149:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290103296/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878837.242154:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242156:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878837.242158:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242159:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878837.242160:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242161:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878837.242173:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878837.242175:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878837.242178:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878837.242179:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878837.242182:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290103296:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878837.242184:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878837.242186:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878837.242187:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878837.242188:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878837.242190:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878837.242191:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878837.242193:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290103424:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878837.242196:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878837.242198:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878837.242199:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878837.242200:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878837.242202:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.242207:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878837.242208:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:0.0:1788878837.242210:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878837.242218:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878837.242221:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878837.242223:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878837.242224:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.242225:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878837.242227:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.242229:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304448522 00000100:00000001:0.0:1788878837.242231:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304449065 00000100:00000001:0.0:1788878837.242233:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304448522 00000100:00000001:0.0:1788878837.242235:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304449065 00000100:00000001:0.0:1788878837.242236:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878837.242237:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.242239:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878837.242242:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878837/real 1788878837] req@ffff94e5045e6a00 x1875765290103424/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878837.242248:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878837.242248:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878837.242250:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878837.242251:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878837.242253:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242254:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878837.242256:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878837.242261:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290103424/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878837.242268:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242271:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878837.242273:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242275:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878837.242277:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242279:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878837.242291:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878837.242295:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878837.242299:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878837.242302:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878837.242306:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290103424:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878837.242319:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878837.242320:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878837.242322:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878837.242324:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878837.242325:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878837.242327:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878837.242329:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290103552:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878837.242334:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878837.242335:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878837.242337:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878837.242338:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878837.242341:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.242345:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878837.242347:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:0.0:1788878837.242349:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878837.242359:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878837.242362:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878837.242364:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878837.242365:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.242366:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878837.242368:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.242371:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304448522 00000100:00000001:0.0:1788878837.242373:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304449065 00000100:00000001:0.0:1788878837.242374:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304448522 00000100:00000001:0.0:1788878837.242376:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304449065 00000100:00000001:0.0:1788878837.242378:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878837.242379:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.242380:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878837.242383:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878837/real 1788878837] req@ffff94e5045e6680 x1875765290103552/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878837.242388:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878837.242389:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878837.242391:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878837.242392:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878837.242393:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242395:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878837.242396:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878837.242398:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290103552/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878837.242409:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242410:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878837.242412:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242413:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878837.242414:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242415:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878837.242424:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878837.242426:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878837.242428:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878837.242430:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878837.242432:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290103552:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878837.242435:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242436:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878837.242437:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878837.242438:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878837.242440:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878837.242441:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878837.242442:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878837.242443:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878837.242444:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878837.242446:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878837.242447:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242448:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878837.242449:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878837.242450:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878837.242452:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878837.242453:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878837.242454:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878837.242455:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878837.242456:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878837.242457:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878837.242458:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242459:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878837.242459:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878837.242461:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878837.242462:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878837.242463:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878837.242464:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878837.242465:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878837.242466:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878837.242467:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878837.242468:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242469:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878837.242472:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878837.242472:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878837.242474:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878837.242475:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878838.009083:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878838.009093:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878838.009100:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878838.009102:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878838.009128:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878838.009132:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878838.009135:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878839.033066:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878839.033073:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878839.033081:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878839.033083:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878839.033137:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878839.033143:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878839.033147:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878840.056981:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878840.057001:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878840.057009:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878840.057010:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878840.057058:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878840.057061:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878840.057065:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878841.081009:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878841.081014:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878841.081018:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878841.081021:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878841.081023:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878841.081026:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878841.081029:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878841.081031:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878841.081033:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878841.081035:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878841.081053:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878841.081055:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878841.081058:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878842.105053:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878842.105062:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878842.105071:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878842.105073:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878842.105104:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878842.105108:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878842.105113:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878842.361067:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878842.361081:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878842.361085:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878842.361091:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878842.361098:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878842.361106:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9781 00000100:00080000:2.0:1788878842.361118:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878842.361122:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878842.361124:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878842.361133:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878842.361136:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878842.361139:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878842.361142:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878842.361144:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878842.361145:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878842.361150:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878842.361153:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878842.361156:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878842.361159:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878842.361162:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878842.361165:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878842.361167:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878842.361179:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878842.361182:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878842.361185:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878842.361187:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878842.361192:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878842.361204:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878842.361213:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290103680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878842.361224:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878842.361229:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878842.361243:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878842.361247:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878842.361250:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878842.361252:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878842.361254:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878842.361256:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878842.361259:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9781 00000100:00080000:2.0:1788878842.361262:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878842.361263:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878842.361265:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878842.361268:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878842.361269:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878842.361287:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878842.361288:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878842.361290:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878842.361291:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878842.361294:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878842.361296:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878842.361298:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878842.361302:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878842.361305:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878842.361306:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878842.361310:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878842.361312:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361314:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878842.361315:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361321:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788878842.361322:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878842.361325:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788878842.361325:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878842.361327:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878842.361330:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788878842.361330:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878842.361335:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878842.361335:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878842.361343:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788878842.361344:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878842.361346:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878842.361350:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878842.361351:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290103808/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1788878842.361358:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290103680:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878842.361362:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878842.361366:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788878842.361367:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878842.361372:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788878842.361373:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878842.361376:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878842.361377:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1788878842.361382:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1788878842.361383:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878842.361386:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878842.361387:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1788878842.361388:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878842.361389:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878842.361394:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9781 00000100:00080000:2.0:1788878842.361397:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878842.361398:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878842.361399:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878842.361401:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878842.361402:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878842.361404:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878842.361405:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1788878842.361406:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878842.361409:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:2.0:1788878842.361410:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878842.361413:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878842.361415:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788878842.361418:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878842.361422:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878842.361423:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878842.361426:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878842.361427:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878842.361429:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878842.361431:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878842.361433:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878842.361434:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878842.361437:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878842.361438:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878842.361439:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878842.361441:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878842.361442:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878842.361447:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878842.361450:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290103936/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878842.361455:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878842.361463:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878842.361465:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878842.361467:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878842.361469:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878842.361471:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788878842.361472:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878842.361475:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878842.361477:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788878842.361477:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878842.361482:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878842.361482:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878842.361487:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:2.0:1788878842.361488:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9781 00000100:00000001:0.0:1788878842.361493:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878842.361494:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878842.361500:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878842.361501:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878842.361502:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304454185 00000020:00000001:2.0:1788878842.361504:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878842.361506:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878842.361507:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878842.361508:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878842.361510:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878842.361512:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878842.361513:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1788878842.361516:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878842.361520:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361521:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304454185 02000000:00000001:2.0:1788878842.361522:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878842.361526:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878842.361528:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361529:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878842.361530:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878842.361535:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878842.361536:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1788878842.361538:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361542:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878842.361543:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361549:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1788878842.361550:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878842.361553:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878842.361556:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:0.0:1788878842.361557:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878842/real 1788878842] req@ffff94e5045e4700 x1875765290103680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788878842.361558:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878842.361561:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878842.361567:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788878842.361573:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:2.0:1788878842.361575:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290104064/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1788878842.361576:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878842.361580:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878842.361584:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878842.361585:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878842.361588:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361589:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878842.361592:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361593:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878842.361596:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878842.361601:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290103680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878842.361608:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.361610:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878842.361612:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.361614:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878842.361615:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.361617:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878842.361619:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878842.361621:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878842.361624:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878842.361625:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878842.361628:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290103680:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878842.361631:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.361633:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878842.361634:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878842.361636:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878842.361640:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878842.361641:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878842.361643:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878842.361646:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878842.361647:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878842.361649:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878842.361650:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.361652:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.361657:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878842.361658:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361660:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878842.361661:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878842.361663:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878842.361664:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878842.361665:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878842.361666:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878842.361668:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878842.361669:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878842.361671:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290103808:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878842.361674:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878842.361675:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878842.361676:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878842.361678:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878842.361680:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361684:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878842.361685:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00000001:0.0:1788878842.361689:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878842.361697:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878842.361700:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878842.361701:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878842.361702:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361703:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878842.361705:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361708:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304453643 00000100:00000001:0.0:1788878842.361710:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304454185 00000100:00000001:0.0:1788878842.361712:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304453643 00000100:00000001:0.0:1788878842.361713:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304454185 00000100:00000001:0.0:1788878842.361715:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878842.361717:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361718:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878842.361721:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878842/real 1788878842] req@ffff94e5045e7480 x1875765290103808/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878842.361727:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878842.361728:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878842.361730:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878842.361731:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878842.361732:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.361733:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878842.361734:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878842.361736:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290103808/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878842.361741:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.361743:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878842.361744:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.361745:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878842.361746:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.361747:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878842.361756:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878842.361758:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878842.361760:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878842.361761:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878842.361763:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290103808:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878842.361766:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878842.361767:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878842.361768:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878842.361769:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878842.361770:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878842.361772:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878842.361774:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290103936:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878842.361776:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878842.361777:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878842.361779:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878842.361780:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878842.361782:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361785:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878842.361786:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00000001:0.0:1788878842.361788:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878842.361794:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878842.361797:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878842.361798:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878842.361799:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361800:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878842.361802:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361804:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304453641 00000100:00000001:0.0:1788878842.361806:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304454185 00000100:00000001:0.0:1788878842.361807:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304453641 00000100:00000001:0.0:1788878842.361809:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304454185 00000100:00000001:0.0:1788878842.361810:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878842.361811:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361813:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878842.361815:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878842/real 1788878842] req@ffff94e5045e7b80 x1875765290103936/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878842.361820:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878842.361821:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878842.361822:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878842.361823:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878842.361824:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.361825:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878842.361826:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878842.361828:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290103936/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878842.361838:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.361839:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878842.361841:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.361842:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878842.361843:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.361844:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878842.361857:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878842.361860:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878842.361864:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878842.361866:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878842.361869:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290103936:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878842.361873:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878842.361874:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878842.361876:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878842.361878:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878842.361881:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878842.361883:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878842.361887:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290104064:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878842.361890:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878842.361893:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878842.361894:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878842.361896:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878842.361900:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361932:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878842.361933:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:0.0:1788878842.361936:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878842.361946:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878842.361949:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878842.361951:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878842.361952:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361954:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878842.361956:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361958:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304453642 00000100:00000001:0.0:1788878842.361960:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304454185 00000100:00000001:0.0:1788878842.361962:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304453642 00000100:00000001:0.0:1788878842.361963:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304454185 00000100:00000001:0.0:1788878842.361965:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878842.361966:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.361967:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878842.361970:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878842/real 1788878842] req@ffff94e5045e4e00 x1875765290104064/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878842.361996:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878842.361997:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878842.361999:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878842.362001:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878842.362002:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.362003:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878842.362020:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878842.362025:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290104064/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878842.362032:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.362034:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878842.362036:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.362038:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878842.362039:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.362041:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878842.362054:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878842.362058:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878842.362062:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878842.362065:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878842.362070:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290104064:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878842.362076:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.362079:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878842.362081:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878842.362083:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878842.362087:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878842.362089:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878842.362091:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878842.362095:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878842.362097:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878842.362100:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878842.362102:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.362105:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878842.362106:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878842.362108:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878842.362112:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878842.362113:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878842.362115:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878842.362118:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878842.362120:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878842.362122:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878842.362124:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.362127:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878842.362128:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878842.362129:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878842.362131:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878842.362132:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878842.362133:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878842.362135:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878842.362136:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878842.362138:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878842.362139:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.362141:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878842.362144:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878842.362146:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878842.362147:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878842.362149:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878843.129026:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878843.129032:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878843.129036:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878843.129038:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878843.129106:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878843.129112:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878843.129117:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878844.153082:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878844.153093:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878844.153096:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878844.153100:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878844.153106:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878844.153108:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878844.153110:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878844.153112:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878844.153115:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878844.153117:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878844.153143:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878844.153146:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878844.153150:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878845.177017:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878845.177025:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878845.177030:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878845.177031:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878845.177052:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878845.177055:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878845.177058:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878846.201064:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878846.201070:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878846.201075:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878846.201076:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878846.201126:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878846.201129:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878846.201132:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878847.225029:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878847.225037:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878847.225042:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878847.225043:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878847.225131:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878847.225136:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878847.225139:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878847.481146:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878847.481154:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878847.481156:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878847.481159:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878847.481163:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878847.481169:0:177900:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9786 00000100:00080000:0.0:1788878847.481178:0:177900:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878847.481180:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878847.481181:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878847.481187:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878847.481189:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878847.481192:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878847.481193:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878847.481195:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878847.481196:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878847.481199:0:177900:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878847.481200:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878847.481202:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878847.481205:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878847.481208:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481210:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878847.481212:0:177900:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878847.481224:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878847.481226:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481229:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878847.481232:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878847.481237:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481248:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878847.481258:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290104192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878847.481269:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878847.481273:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481291:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878847.481293:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878847.481296:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878847.481297:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878847.481298:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878847.481301:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878847.481303:0:177900:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9786 00000100:00080000:0.0:1788878847.481305:0:177900:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878847.481307:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878847.481308:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878847.481310:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878847.481312:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878847.481314:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788878847.481315:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878847.481317:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878847.481318:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878847.481321:0:177900:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878847.481323:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878847.481324:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878847.481329:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878847.481330:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481331:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878847.481335:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878847.481337:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481339:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878847.481343:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878847.481346:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481349:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878847.481354:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878847.481356:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481364:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878847.481373:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979180 x1875765290104320/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878847.481390:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878847.481394:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481399:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878847.481402:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878847.481407:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878847.481408:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878847.481412:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878847.481415:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878847.481420:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9786 00000100:00080000:0.0:1788878847.481426:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878847.481427:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878847.481428:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878847.481431:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878847.481434:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878847.481435:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878847.481437:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878847.481438:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878847.481440:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878847.481442:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878847.481443:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878847.481444:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878847.481446:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878847.481448:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481452:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878847.481454:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878847.481458:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481461:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878847.481464:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878847.481467:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481468:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878847.481469:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878847.481471:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481475:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878847.481478:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290104448/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878847.481488:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878847.481490:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481491:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878847.481493:0:177900:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878847.481495:0:177900:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878847.481495:0:177900:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878847.481496:0:177900:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878847.481498:0:177900:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878847.481500:0:177900:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9786 00000100:00080000:0.0:1788878847.481503:0:177900:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878847.481506:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878847.481507:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878847.481512:0:177900:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878847.481513:0:177900:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878847.481515:0:177900:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788878847.481517:0:177900:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878847.481518:0:177900:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878847.481519:0:177900:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878847.481522:0:177900:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878847.481524:0:177900:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878847.481528:0:177900:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878847.481530:0:177900:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878847.481532:0:177900:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481535:0:177900:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878847.481538:0:177900:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878847.481544:0:177900:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481548:0:177900:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878847.481551:0:177900:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878847.481552:0:177900:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481554:0:177900:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878847.481556:0:177900:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878847.481559:0:177900:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481563:0:177900:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878847.481568:0:177900:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290104576/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878847.481573:0:177900:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878847.481574:0:177900:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878847.481576:0:177900:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878848.249092:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878848.249103:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878848.249113:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878848.249115:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878848.249154:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878848.249159:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878848.249164:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878849.273021:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878849.273028:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878849.273033:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878849.273035:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878849.273062:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878849.273064:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878849.273067:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878850.297037:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878850.297044:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878850.297050:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878850.297051:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878850.297076:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878850.297078:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878850.297082:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878851.321053:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878851.321063:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878851.321072:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878851.321095:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878851.321098:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878851.321104:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878851.321109:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878851.321114:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878851.321118:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878851.321122:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878851.321250:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878851.321257:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878851.321263:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878852.344974:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878852.344979:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878852.344982:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878852.344984:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878852.345001:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878852.345004:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878852.345007:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878852.601251:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878852.601258:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878852.601264:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878852.601268:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878852.601272:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878852.601279:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878852.601284:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878852.601289:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878852.601298:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290104704:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878852.601307:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878852.601313:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878852.601317:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878852.601323:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878852.601331:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.601351:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878852.601355:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286517888 : -117763423033728 : ffff94e51100a680) 00000100:00000001:0.0:1788878852.601363:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878852.601405:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878852.601419:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878852.601427:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878852.601431:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.601436:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878852.601442:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.601451:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304464425 00000100:00000001:0.0:1788878852.601464:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878852.601472:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304464425 00000100:00000001:0.0:1788878852.601484:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878852.601491:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878852.601496:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.601502:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878852.601512:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878852/real 1788878852] req@ffff94e51100a680 x1875765290104704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878852.601540:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878852.601544:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878852.601549:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878852.601554:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878852.601558:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.601562:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878852.601565:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878852.601574:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100a680 x1875765290104704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878852.601596:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.601603:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878852.601608:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.601612:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878852.601616:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.601625:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878852.601629:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878852.601636:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878852.601645:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878852.601651:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878852.601658:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290104704:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878852.601669:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.601673:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878852.601676:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878852.601680:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878852.601685:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878852.601689:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878852.601693:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878852.601699:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878852.601702:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878852.601708:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878852.601711:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.601713:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.601721:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878852.601724:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.601726:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878852.601729:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878852.601730:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878852.601732:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878852.601734:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878852.601736:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878852.601738:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878852.601740:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878852.601744:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290104832:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878852.601747:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878852.601749:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878852.601750:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878852.601752:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878852.601755:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.601759:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878852.601761:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:0.0:1788878852.601765:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878852.601785:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878852.601790:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878852.601792:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878852.601793:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.601795:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878852.601797:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.601801:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304463883 00000100:00000001:0.0:1788878852.601804:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304464425 00000100:00000001:0.0:1788878852.601806:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304463883 00000100:00000001:0.0:1788878852.601808:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304464425 00000100:00000001:0.0:1788878852.601810:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878852.601812:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.601814:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878852.601817:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878852/real 1788878852] req@ffff94e51100b800 x1875765290104832/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878852.601825:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878852.601826:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878852.601828:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878852.601830:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878852.601831:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.601832:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878852.601834:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878852.601837:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290104832/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878852.601843:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.601844:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878852.601846:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.601848:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878852.601849:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.601850:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878852.601861:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878852.601864:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878852.601867:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878852.601869:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878852.601872:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290104832:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878852.601875:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878852.601876:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878852.601877:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878852.601879:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878852.601880:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878852.601882:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878852.601884:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290104960:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878852.601887:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878852.601888:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878852.601890:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878852.601891:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878852.601894:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.601898:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878852.601899:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:0.0:1788878852.601940:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878852.601955:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878852.601959:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878852.601961:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878852.601962:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.601964:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878852.601966:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.601969:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304463882 00000100:00000001:0.0:1788878852.601971:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304464425 00000100:00000001:0.0:1788878852.601973:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304463882 00000100:00000001:0.0:1788878852.601975:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304464425 00000100:00000001:0.0:1788878852.601977:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878852.601979:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.601980:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878852.601984:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878852/real 1788878852] req@ffff94e51100b480 x1875765290104960/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878852.601990:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878852.601991:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878852.601994:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878852.601995:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878852.601996:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.601998:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878852.602000:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878852.602005:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290104960/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878852.602023:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.602026:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878852.602029:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.602031:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878852.602033:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.602036:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878852.602051:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878852.602055:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878852.602060:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878852.602063:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878852.602068:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290104960:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878852.602073:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878852.602075:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878852.602077:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878852.602080:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878852.602083:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878852.602086:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878852.602090:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290105088:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878852.602094:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878852.602097:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878852.602099:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878852.602102:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878852.602106:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.602114:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878852.602116:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:0.0:1788878852.602121:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878852.602137:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878852.602141:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878852.602144:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878852.602145:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.602147:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878852.602149:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.602152:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304463882 00000100:00000001:0.0:1788878852.602155:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304464425 00000100:00000001:0.0:1788878852.602157:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304463882 00000100:00000001:0.0:1788878852.602159:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304464425 00000100:00000001:0.0:1788878852.602160:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878852.602162:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.602164:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878852.602167:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878852/real 1788878852] req@ffff94e51100aa00 x1875765290105088/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878852.602174:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878852.602175:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878852.602177:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878852.602178:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878852.602180:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.602181:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878852.602182:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878852.602186:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290105088/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878852.602191:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.602193:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878852.602195:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.602196:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878852.602197:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.602198:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878852.602208:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878852.602211:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878852.602213:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878852.602215:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878852.602218:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290105088:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878852.602222:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.602223:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878852.602224:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878852.602226:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878852.602228:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878852.602229:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878852.602230:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878852.602232:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878852.602233:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878852.602235:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878852.602236:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.602238:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878852.602239:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878852.602240:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878852.602241:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878852.602242:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878852.602243:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878852.602244:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878852.602245:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878852.602246:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878852.602247:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.602248:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878852.602249:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878852.602250:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878852.602252:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878852.602253:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878852.602253:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878852.602255:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878852.602256:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878852.602257:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878852.602258:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.602259:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878852.602262:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878852.602263:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878852.602265:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878852.602266:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878853.369060:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878853.369071:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878853.369082:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878853.369086:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878853.369124:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878853.369128:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878853.369132:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878854.393034:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878854.393044:0:178018:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878854.393047:0:178018:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878854.393050:0:178018:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878854.393056:0:178018:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878854.393058:0:178018:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878854.393060:0:178018:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878854.393062:0:178018:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878854.393065:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878854.393067:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878854.393090:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878854.393093:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878854.393095:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878855.417063:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878855.417073:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878855.417092:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878855.417093:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878855.417195:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878855.417199:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878855.417203:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878856.441030:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878856.441037:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878856.441042:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878856.441043:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878856.441068:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878856.441071:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878856.441074:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878857.465033:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878857.465038:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878857.465043:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878857.465044:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878857.465071:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878857.465073:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878857.465076:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878857.721082:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878857.721100:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878857.721104:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878857.721110:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878857.721116:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878857.721128:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9796 00000100:00080000:2.0:1788878857.721145:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878857.721150:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878857.721153:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878857.721165:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878857.721169:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878857.721176:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878857.721179:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878857.721182:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878857.721184:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878857.721191:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878857.721194:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878857.721199:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878857.721204:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878857.721209:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878857.721212:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878857.721215:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878857.721230:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878857.721232:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878857.721237:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878857.721241:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878857.721248:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878857.721268:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878857.721282:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4000 x1875765290105216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878857.721297:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878857.721304:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878857.721327:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878857.721332:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878857.721337:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878857.721339:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878857.721341:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878857.721345:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878857.721349:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9796 00000100:00080000:2.0:1788878857.721353:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878857.721355:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878857.721357:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878857.721361:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878857.721364:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878857.721367:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878857.721370:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878857.721373:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878857.721374:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878857.721378:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878857.721381:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878857.721383:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878857.721391:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878857.721394:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878857.721396:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878857.721403:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878857.721407:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878857.721411:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878857.721416:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878857.721419:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878857.721421:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878857.721423:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878857.721426:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878857.721436:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.721441:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1788878857.721449:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290105344/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878857.721451:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878857.721456:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878857.721462:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878857.721467:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878857.721477:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788878857.721482:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1788878857.721484:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878857.721491:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878857.721495:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878857.721503:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290105216:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878857.721509:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.721512:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1788878857.721521:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1788878857.721523:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878857.721529:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878857.721538:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1788878857.721540:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878857.721546:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:0.0:1788878857.721548:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878857.721550:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878857.721560:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878857.721566:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9796 00000100:00000001:0.0:1788878857.721581:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878857.721590:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074548224 : -117763635003392 : ffff94e5045e4000) 00000100:00080000:2.0:1788878857.721590:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878857.721599:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878857.721605:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878857.721606:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1788878857.721612:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878857.721617:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878857.721620:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878857.721624:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878857.721627:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878857.721629:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878857.721634:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878857.721638:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878857.721640:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878857.721644:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878857.721648:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878857.721650:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878857.721654:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878857.721658:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878857.721661:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878857.721670:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878857.721672:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878857.721674:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878857.721676:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878857.721680:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788878857.721694:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878857.721694:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878857.721709:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290105472/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878857.721721:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878857.721724:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878857.721725:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878857.721735:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.721739:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788878857.721740:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878857.721748:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878857.721749:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878857.721754:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878857.721760:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:2.0:1788878857.721760:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878857.721769:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878857.721773:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9796 00000100:00000001:0.0:1788878857.721776:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878857.721780:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878857.721787:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878857.721789:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878857.721791:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304469545 00000020:00000001:2.0:1788878857.721793:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878857.721798:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878857.721802:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878857.721805:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878857.721807:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878857.721809:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878857.721814:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878857.721816:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878857.721818:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878857.721826:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878857.721832:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878857.721834:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304469545 02000000:00000001:2.0:1788878857.721838:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.721845:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878857.721847:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878857.721852:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00100000:2.0:1788878857.721855:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878857.721859:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788878857.721894:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.721895:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878857.721946:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878857/real 1788878857] req@ffff94e5045e4000 x1875765290105216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878857.721964:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788878857.721968:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878857.721973:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1788878857.721975:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878857.721980:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.721982:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878857.721984:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878857.721990:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878857.721991:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:2.0:1788878857.722000:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722001:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722012:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878857.722016:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878857.722018:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878857.722026:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4000 x1875765290105216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878857.722037:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290105600/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878857.722045:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722055:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878857.722062:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878857.722064:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878857.722071:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722072:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878857.722076:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878857.722082:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722083:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878857.722090:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878857.722095:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878857.722102:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878857.722106:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878857.722113:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290105216:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878857.722120:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722124:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878857.722127:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878857.722131:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878857.722136:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878857.722139:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878857.722142:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878857.722148:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878857.722150:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878857.722153:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878857.722155:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722158:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722169:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878857.722172:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722175:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878857.722177:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878857.722179:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878857.722181:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878857.722183:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878857.722186:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878857.722190:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878857.722193:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878857.722197:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290105344:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878857.722201:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878857.722204:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878857.722206:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878857.722208:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878857.722213:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722221:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878857.722223:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074559872 : -117763634991744 : ffff94e5045e6d80) 00000100:00000001:0.0:1788878857.722230:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878857.722249:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878857.722255:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878857.722258:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878857.722260:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722262:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878857.722266:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722271:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304469004 00000100:00000001:0.0:1788878857.722275:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304469545 00000100:00000001:0.0:1788878857.722278:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304469004 00000100:00000001:0.0:1788878857.722281:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304469545 00000100:00000001:0.0:1788878857.722284:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878857.722286:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722289:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878857.722294:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878857/real 1788878857] req@ffff94e5045e6d80 x1875765290105344/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878857.722304:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878857.722305:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878857.722308:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878857.722310:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878857.722312:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722314:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878857.722316:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878857.722321:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6d80 x1875765290105344/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878857.722331:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722334:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878857.722336:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722338:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878857.722340:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722342:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878857.722363:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878857.722366:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878857.722369:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878857.722372:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878857.722376:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290105344:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878857.722380:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878857.722382:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878857.722384:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878857.722387:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878857.722389:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878857.722392:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878857.722395:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290105472:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878857.722399:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878857.722401:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878857.722403:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878857.722405:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878857.722409:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722415:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878857.722417:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:0.0:1788878857.722422:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878857.722433:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878857.722438:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878857.722441:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878857.722443:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722445:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878857.722448:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722452:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304469002 00000100:00000001:0.0:1788878857.722455:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304469545 00000100:00000001:0.0:1788878857.722458:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304469002 00000100:00000001:0.0:1788878857.722462:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304469545 00000100:00000001:0.0:1788878857.722464:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878857.722466:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722468:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878857.722473:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878857/real 1788878857] req@ffff94e5045e4e00 x1875765290105472/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878857.722482:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878857.722483:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878857.722486:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878857.722487:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878857.722489:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722492:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878857.722493:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878857.722498:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290105472/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878857.722507:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722509:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878857.722511:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722513:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878857.722515:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722517:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878857.722535:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878857.722538:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878857.722541:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878857.722544:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878857.722548:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290105472:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878857.722553:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878857.722555:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878857.722557:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878857.722559:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878857.722561:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878857.722563:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878857.722566:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290105600:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878857.722570:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878857.722572:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878857.722574:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878857.722575:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878857.722579:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722584:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878857.722585:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00000001:0.0:1788878857.722589:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878857.722600:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878857.722604:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878857.722606:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878857.722608:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722610:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878857.722613:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722617:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304469002 00000100:00000001:0.0:1788878857.722620:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304469545 00000100:00000001:0.0:1788878857.722622:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304469002 00000100:00000001:0.0:1788878857.722625:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304469545 00000100:00000001:0.0:1788878857.722627:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878857.722629:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722631:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878857.722635:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878857/real 1788878857] req@ffff94e5045e7b80 x1875765290105600/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878857.722653:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878857.722655:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878857.722657:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878857.722659:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878857.722661:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722663:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878857.722664:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878857.722669:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290105600/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878857.722676:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722679:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878857.722681:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722683:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878857.722685:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722687:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878857.722705:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878857.722708:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878857.722711:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878857.722713:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878857.722717:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290105600:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878857.722720:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722723:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878857.722724:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878857.722726:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878857.722728:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878857.722730:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878857.722731:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878857.722734:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878857.722736:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878857.722739:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878857.722741:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722743:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878857.722745:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878857.722747:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878857.722749:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878857.722751:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878857.722753:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878857.722755:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878857.722756:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878857.722758:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878857.722760:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722761:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878857.722763:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878857.722764:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878857.722766:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878857.722767:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878857.722768:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878857.722771:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878857.722772:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878857.722774:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878857.722775:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722777:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878857.722780:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878857.722782:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878857.722785:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878857.722786:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878858.489090:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878858.489100:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878858.489108:0:178018:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878858.489110:0:178018:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878858.489143:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878858.489147:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878858.489151:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878859.513064:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878859.513073:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878859.513080:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878859.513082:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878859.513194:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878859.513201:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878859.513207:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878860.537043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878860.537052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878860.537059:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878860.537062:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878860.537086:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878860.537090:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878860.537094:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878861.561049:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878861.561058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878861.561065:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878861.561069:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878861.561073:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878861.561077:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878861.561080:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878861.561082:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878861.561085:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878861.561087:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878861.561112:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878861.561115:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878861.561118:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:0.0:1788878862.585032:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878862.585042:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878862.585051:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878862.585053:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878862.585098:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878862.585102:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878862.585120:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878862.841033:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878862.841045:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878862.841048:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878862.841053:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878862.841059:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878862.841068:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9801 00000100:00080000:0.0:1788878862.841081:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878862.841085:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878862.841088:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878862.841098:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878862.841111:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878862.841115:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878862.841118:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878862.841121:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878862.841123:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878862.841128:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878862.841131:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878862.841134:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878862.841138:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878862.841142:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878862.841145:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878862.841147:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878862.841165:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878862.841168:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878862.841173:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878862.841177:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878862.841185:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878862.841208:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878862.841224:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290105728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878862.841238:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878862.841244:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878862.841266:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878862.841270:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878862.841274:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878862.841275:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878862.841277:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878862.841281:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878862.841284:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9801 00000100:00080000:0.0:1788878862.841290:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878862.841293:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878862.841295:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878862.841299:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878862.841301:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878862.841306:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788878862.841312:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878862.841315:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878862.841321:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878862.841329:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878862.841335:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878862.841339:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878862.841347:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878862.841354:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878862.841359:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878862.841365:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878862.841370:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878862.841374:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878862.841380:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878862.841384:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878862.841389:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878862.841393:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878862.841397:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878862.841407:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878862.841418:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290105856/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878862.841433:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878862.841441:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878862.841449:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878862.841456:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878862.841462:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878862.841465:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878862.841467:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878862.841472:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878862.841476:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9801 00000100:00080000:0.0:1788878862.841492:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878862.841495:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878862.841497:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878862.841501:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878862.841503:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878862.841508:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878862.841512:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878862.841517:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878862.841518:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878862.841523:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878862.841527:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878862.841528:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878862.841532:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878862.841536:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878862.841539:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878862.841544:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878862.841549:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878862.841553:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878862.841559:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878862.841563:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878862.841566:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878862.841570:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878862.841574:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878862.841582:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878862.841589:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979180 x1875765290105984/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878862.841603:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878862.841607:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878862.841608:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878862.841616:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.841617:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1788878862.841620:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788878862.841622:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878862.841630:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788878862.841631:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788878862.841633:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878862.841637:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878862.841641:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00100000:1.0:1788878862.841643:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878862/real 1788878862] req@ffff94e508978380 x1875765290105728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878862.841648:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9801 00000100:00080000:0.0:1788878862.841656:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878862.841663:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000400:00000001:1.0:1788878862.841667:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788878862.841668:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:1.0:1788878862.841672:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:0.0:1788878862.841678:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788878862.841679:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000020:00000001:0.0:1788878862.841684:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788878862.841685:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000020:00000001:0.0:1788878862.841691:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788878862.841692:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.841698:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1788878862.841701:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788878862.841702:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878862.841711:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1788878862.841715:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290105728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878862.841717:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878862.841731:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878862.841734:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878862.841737:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878862.841741:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878862.841741:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1788878862.841748:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878862.841748:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788878862.841755:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.841756:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878862.841761:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878862.841762:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788878862.841766:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000008:00100000:0.0:1788878862.841767:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878862.841771:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:0.0:1788878862.841772:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878862.841778:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788878862.841778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:0.0:1788878862.841788:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00080000:1.0:1788878862.841789:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 02000000:00000001:0.0:1788878862.841791:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.841794:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1788878862.841795:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878862.841800:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1788878862.841804:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290105728:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788878862.841806:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878862.841817:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.841817:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.841821:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878862.841825:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00100000:0.0:1788878862.841830:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290106112/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788878862.841852:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878862.841858:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878862.841862:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878862.841866:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:0.0:1788878862.841892:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788878862.841892:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878862.841896:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878862.841900:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.841901:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878862.841905:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878862.841909:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.841910:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.841961:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878862.841964:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.841967:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878862.841977:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878862.841979:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878862.841981:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878862.841983:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878862.841991:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878862.841992:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878862.841995:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878862.841999:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290105856:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878862.842003:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878862.842005:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878862.842007:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878862.842008:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878862.842012:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.842020:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878862.842022:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000100:00000001:1.0:1788878862.842028:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878862.842042:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878862.842047:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878862.842049:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878862.842050:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.842052:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878862.842054:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.842059:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304474124 00000100:00000001:1.0:1788878862.842062:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304474665 00000100:00000001:1.0:1788878862.842064:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304474124 00000100:00000001:1.0:1788878862.842066:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304474665 00000100:00000001:1.0:1788878862.842068:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878862.842071:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.842073:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878862.842077:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878862/real 1788878862] req@ffff94e50897b800 x1875765290105856/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878862.842091:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878862.842093:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878862.842097:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878862.842099:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878862.842100:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842103:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878862.842105:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878862.842111:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290105856/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878862.842122:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842126:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878862.842130:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842132:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878862.842134:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842137:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878862.842158:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878862.842161:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878862.842165:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878862.842168:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878862.842171:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290105856:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878862.842175:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878862.842177:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878862.842178:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878862.842180:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878862.842182:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878862.842184:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878862.842187:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290105984:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878862.842191:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878862.842192:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878862.842194:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878862.842196:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878862.842199:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.842206:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878862.842207:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 00000100:00000001:1.0:1788878862.842211:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878862.842224:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878862.842228:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878862.842230:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878862.842232:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.842233:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878862.842236:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.842239:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304474122 00000100:00000001:1.0:1788878862.842242:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304474665 00000100:00000001:1.0:1788878862.842244:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304474122 00000100:00000001:1.0:1788878862.842246:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304474665 00000100:00000001:1.0:1788878862.842248:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878862.842249:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.842251:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878862.842254:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878862/real 1788878862] req@ffff94e508979180 x1875765290105984/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878862.842261:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878862.842262:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878862.842264:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878862.842266:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878862.842267:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842269:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878862.842270:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878862.842274:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290105984/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878862.842287:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842289:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878862.842291:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842292:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878862.842293:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842295:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878862.842309:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878862.842311:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878862.842313:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878862.842315:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878862.842318:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290105984:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878862.842321:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878862.842323:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878862.842324:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878862.842326:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878862.842328:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878862.842330:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878862.842332:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290106112:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878862.842334:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878862.842336:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878862.842337:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878862.842338:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878862.842341:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.842344:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878862.842345:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:1.0:1788878862.842348:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878862.842356:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878862.842359:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878862.842361:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878862.842362:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.842364:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878862.842367:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.842369:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304474122 00000100:00000001:1.0:1788878862.842372:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304474665 00000100:00000001:1.0:1788878862.842374:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304474122 00000100:00000001:1.0:1788878862.842376:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304474665 00000100:00000001:1.0:1788878862.842378:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878862.842380:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.842381:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878862.842384:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878862/real 1788878862] req@ffff94e508979c00 x1875765290106112/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878862.842391:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878862.842392:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878862.842393:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878862.842394:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878862.842396:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842397:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878862.842398:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878862.842403:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290106112/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878862.842414:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842416:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878862.842420:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842422:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878862.842423:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842426:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878862.842444:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878862.842449:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878862.842454:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878862.842457:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878862.842463:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290106112:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878862.842469:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842471:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878862.842473:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878862.842474:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878862.842477:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878862.842478:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878862.842480:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878862.842482:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878862.842483:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878862.842487:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878862.842488:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842489:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878862.842491:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878862.842492:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878862.842493:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878862.842494:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878862.842495:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878862.842497:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878862.842499:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878862.842501:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878862.842502:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842504:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878862.842505:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878862.842506:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878862.842507:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878862.842508:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878862.842509:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878862.842510:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878862.842511:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878862.842512:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878862.842513:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842514:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.842518:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878862.842519:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.842521:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878862.842522:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878862.904957:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878862.904964:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878862.904966:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878862.904971:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878862.904974:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878862.904978:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.904981:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878862.904983:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878862.904986:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878863.608997:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878863.609006:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878863.609013:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878863.609015:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878863.609075:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878863.609079:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878863.609082:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878863.929001:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878863.929010:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878863.929012:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878863.929017:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878863.929021:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878863.929025:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878863.929028:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878863.929031:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878863.929033:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878864.633015:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878864.633024:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878864.633027:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878864.633030:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878864.633034:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878864.633036:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878864.633038:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878864.633040:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878864.633043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878864.633044:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878864.633064:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878864.633067:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878864.633070:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878864.953014:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878864.953020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878864.953022:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878864.953027:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878864.953030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878864.953034:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878864.953036:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878864.953039:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878864.953058:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878865.656983:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878865.656990:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878865.657003:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878865.657004:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878865.657026:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878865.657029:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878865.657031:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878865.977021:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878865.977031:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878865.977034:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878865.977041:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878865.977045:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878865.977050:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878865.977054:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878865.977059:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878865.977063:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878866.680998:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878866.681005:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878866.681013:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878866.681015:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878866.681036:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878866.681039:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878866.681043:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878867.001013:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878867.001021:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878867.001024:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878867.001030:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878867.001035:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878867.001040:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.001043:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878867.001046:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.001049:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878867.705007:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878867.705012:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878867.705016:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878867.705017:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878867.705084:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878867.705088:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878867.705093:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878867.961081:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878867.961093:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878867.961096:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878867.961101:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878867.961106:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878867.961115:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9806 00000100:00080000:2.0:1788878867.961130:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878867.961133:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878867.961135:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878867.961147:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878867.961150:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878867.961171:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878867.961174:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878867.961176:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878867.961177:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878867.961182:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878867.961184:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878867.961199:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878867.961204:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878867.961207:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878867.961209:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878867.961211:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878867.961229:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878867.961230:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878867.961234:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878867.961237:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878867.961243:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878867.961258:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878867.961269:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290106240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878867.961281:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878867.961286:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878867.961307:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878867.961311:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878867.961315:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878867.961317:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878867.961318:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878867.961321:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878867.961323:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9806 00000100:00080000:2.0:1788878867.961326:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878867.961327:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878867.961329:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878867.961332:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878867.961333:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878867.961335:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878867.961337:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878867.961338:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878867.961339:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878867.961342:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878867.961343:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878867.961345:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878867.961351:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878867.961352:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878867.961354:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878867.961360:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878867.961363:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878867.961364:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878867.961368:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878867.961369:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878867.961370:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878867.961371:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878867.961373:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878867.961378:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878867.961383:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290106368/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878867.961396:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878867.961398:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878867.961401:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878867.961403:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878867.961405:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878867.961407:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878867.961410:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878867.961411:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878867.961413:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878867.961428:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9806 00000100:00000001:1.0:1788878867.961429:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878867.961440:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1788878867.961441:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878867.961451:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878867.961452:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878867.961454:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878867.961458:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878867.961459:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878867.961464:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878867.961469:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 02000000:00000001:1.0:1788878867.961470:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1788878867.961480:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 02000000:00000001:1.0:1788878867.961482:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878867.961485:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788878867.961489:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878867.961489:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878867.961510:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878867.961518:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878867.961519:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290106240:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878867.961526:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878867.961536:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878867.961536:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878867.961549:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878867.961551:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788878867.961552:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:2.0:1788878867.961554:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788878867.961556:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788878867.961561:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878867.961562:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788878867.961569:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878867.961575:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878867.961576:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878867.961578:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878867.961580:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878867.961584:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878867.961587:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878867.961595:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878867.961601:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290106496/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878867.961603:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878867.961610:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00000001:2.0:1788878867.961626:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878867.961628:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878867.961632:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878867.961638:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878867.961641:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878867.961646:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878867.961648:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878867.961650:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878867.961652:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878867.961654:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9806 00000100:00080000:2.0:1788878867.961658:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878867.961660:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878867.961662:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878867.961665:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878867.961666:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878867.961667:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878867.961669:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878867.961671:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878867.961673:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:1.0:1788878867.961679:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788878867.961686:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878867.961691:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878867.961695:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878867.961699:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878867.961700:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:2.0:1788878867.961702:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878867.961706:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878867.961709:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878867.961710:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878867.961715:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788878867.961717:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.961720:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878867.961722:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.961732:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878867.961732:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878867.961744:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.961746:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304479785 00000100:00000001:2.0:1788878867.961748:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878867.961754:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878867.961757:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.961763:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878867.961763:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878867.961777:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290106624/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878867.961784:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878867.961786:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878867.961789:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.961790:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304479785 00000100:00000001:1.0:1788878867.961849:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788878867.961855:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878867.961859:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.961863:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878867.961873:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878867/real 1788878867] req@ffff94e5045e7480 x1875765290106240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878867.961893:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878867.961895:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878867.961899:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878867.961902:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878867.961904:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.961907:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878867.961909:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878867.961913:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290106240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878867.961949:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.961953:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878867.961956:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.961958:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878867.961959:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.961962:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878867.961964:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878867.961967:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878867.961971:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878867.961974:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878867.961978:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290106240:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878867.961982:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878867.961984:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878867.961986:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878867.961988:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878867.961990:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878867.961992:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878867.961995:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290106368:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878867.961998:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878867.962001:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878867.962003:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878867.962005:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878867.962008:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.962014:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878867.962015:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:1.0:1788878867.962020:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878867.962034:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878867.962039:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878867.962041:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878867.962042:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.962044:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878867.962047:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.962051:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304479244 00000100:00000001:1.0:1788878867.962054:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304479785 00000100:00000001:1.0:1788878867.962056:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304479244 00000100:00000001:1.0:1788878867.962058:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304479785 00000100:00000001:1.0:1788878867.962060:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878867.962062:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.962064:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878867.962068:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878867/real 1788878867] req@ffff94e5045e4700 x1875765290106368/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878867.962075:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878867.962076:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878867.962078:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878867.962080:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878867.962081:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962083:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878867.962084:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878867.962088:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290106368/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878867.962094:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962096:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878867.962098:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962100:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878867.962101:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962103:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878867.962116:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878867.962118:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878867.962121:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878867.962124:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878867.962126:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290106368:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878867.962130:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962132:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878867.962133:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878867.962136:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878867.962139:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878867.962141:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878867.962143:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878867.962147:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878867.962149:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878867.962163:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878867.962165:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962166:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878867.962167:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878867.962168:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878867.962170:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878867.962171:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878867.962172:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878867.962173:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878867.962174:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878867.962176:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878867.962177:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962179:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962202:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878867.962204:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.962207:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878867.962209:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878867.962210:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878867.962212:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878867.962214:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878867.962217:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878867.962219:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878867.962221:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878867.962225:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290106496:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878867.962228:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878867.962230:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878867.962232:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878867.962234:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878867.962237:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.962243:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878867.962245:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:1.0:1788878867.962249:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878867.962261:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878867.962265:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878867.962267:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878867.962269:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.962271:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878867.962274:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.962277:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304479242 00000100:00000001:1.0:1788878867.962280:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304479785 00000100:00000001:1.0:1788878867.962283:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304479242 00000100:00000001:1.0:1788878867.962285:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304479785 00000100:00000001:1.0:1788878867.962287:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878867.962289:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.962291:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878867.962295:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878867/real 1788878867] req@ffff94e5045e6680 x1875765290106496/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878867.962303:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878867.962304:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878867.962306:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878867.962308:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878867.962309:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962311:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878867.962313:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878867.962317:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290106496/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878867.962325:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962327:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878867.962329:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962331:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878867.962332:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962334:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878867.962347:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878867.962349:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878867.962352:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878867.962354:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878867.962357:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290106496:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878867.962361:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878867.962362:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878867.962364:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878867.962367:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878867.962369:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878867.962371:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878867.962374:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290106624:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878867.962377:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878867.962379:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878867.962381:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878867.962382:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878867.962385:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.962389:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878867.962390:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:1.0:1788878867.962393:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878867.962402:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878867.962406:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878867.962408:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878867.962410:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.962412:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878867.962414:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.962417:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304479242 00000100:00000001:1.0:1788878867.962420:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304479785 00000100:00000001:1.0:1788878867.962422:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304479242 00000100:00000001:1.0:1788878867.962425:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304479785 00000100:00000001:1.0:1788878867.962427:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878867.962429:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.962431:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878867.962435:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878867/real 1788878867] req@ffff94e5045e6a00 x1875765290106624/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878867.962442:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878867.962443:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878867.962446:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878867.962447:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878867.962448:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962450:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878867.962451:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878867.962455:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290106624/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878867.962462:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962464:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878867.962467:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962468:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878867.962470:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962472:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878867.962484:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878867.962486:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878867.962490:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878867.962492:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878867.962496:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290106624:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878867.962500:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962501:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878867.962503:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878867.962504:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878867.962506:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878867.962507:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878867.962509:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878867.962511:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878867.962512:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878867.962514:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878867.962515:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962517:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878867.962518:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878867.962520:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878867.962521:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878867.962522:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878867.962524:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878867.962525:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878867.962527:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878867.962528:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878867.962529:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962531:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878867.962534:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878867.962535:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878867.962537:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878867.962539:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878868.024978:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878868.024983:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878868.024985:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878868.024990:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878868.025000:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878868.025002:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878868.025004:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878868.025007:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878868.025009:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878868.729036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878868.729042:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878868.729048:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878868.729049:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878868.729069:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878868.729072:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878868.729075:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878869.049011:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878869.049020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878869.049023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878869.049029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878869.049034:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878869.049039:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878869.049042:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878869.049045:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878869.049048:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878869.753018:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878869.753025:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878869.753029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878869.753031:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878869.753051:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878869.753053:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878869.753056:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878870.072974:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878870.072981:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878870.072982:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878870.072986:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878870.072989:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878870.072992:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878870.072994:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878870.072996:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878870.072998:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878870.777007:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878870.777013:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878870.777017:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878870.777018:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878870.777034:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878870.777036:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878870.777038:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878871.096961:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878871.096967:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878871.096968:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878871.096972:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878871.096975:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878871.096977:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878871.096979:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878871.096981:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878871.096984:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878871.801001:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878871.801006:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878871.801010:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878871.801012:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878871.801014:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878871.801017:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878871.801019:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878871.801020:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878871.801022:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878871.801023:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878871.801080:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878871.801083:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878871.801086:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878872.120988:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878872.120995:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878872.120997:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878872.121002:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878872.121005:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878872.121008:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878872.121011:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878872.121013:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878872.121015:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878872.825012:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878872.825018:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878872.825022:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878872.825023:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878872.825053:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878872.825055:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878872.825057:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878873.081049:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878873.081064:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878873.081068:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878873.081074:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878873.081085:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878873.081100:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9811 00000100:00080000:2.0:1788878873.081122:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878873.081130:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878873.081135:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878873.081153:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878873.081177:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878873.081185:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878873.081192:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878873.081197:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878873.081201:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878873.081212:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878873.081218:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878873.081222:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878873.081237:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878873.081242:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878873.081246:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878873.081252:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878873.081271:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878873.081275:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878873.081280:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878873.081286:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878873.081294:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878873.081313:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878873.081328:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290106752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878873.081344:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878873.081351:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878873.081376:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878873.081382:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878873.081389:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878873.081392:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878873.081395:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878873.081400:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878873.081406:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9811 00000100:00000001:1.0:1788878873.081412:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788878873.081415:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878873.081420:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878873.081422:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878873.081423:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878873.081434:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788878873.081435:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878873.081443:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788878873.081448:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1788878873.081448:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878873.081461:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788878873.081462:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878873.081472:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878873.081475:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788878873.081479:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1788878873.081484:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788878873.081487:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878873.081490:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878873.081497:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878873.081498:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878873.081516:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788878873.081517:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290106752:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878873.081526:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.081531:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788878873.081532:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788878873.081541:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:2.0:1788878873.081542:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788878873.081547:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788878873.081551:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878873.081553:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788878873.081561:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878873.081563:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878873.081569:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878873.081573:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878873.081576:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878873.081578:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788878873.081584:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788878873.081584:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.081590:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:2.0:1788878873.081595:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.081596:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1788878873.081612:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290106880/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878873.081623:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878873.081627:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878873.081632:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878873.081637:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878873.081647:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1788878873.081648:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878873.081654:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878873.081658:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878873.081664:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788878873.081667:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788878873.081673:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9811 00000100:00000001:1.0:1788878873.081678:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878873.081682:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878873.081682:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878873.081695:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878873.081696:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878873.081706:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878873.081710:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878873.081715:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878873.081718:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788878873.081719:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304484905 00000020:00000001:2.0:1788878873.081723:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878873.081729:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878873.081734:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878873.081735:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878873.081737:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878873.081746:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304484905 00000100:00080000:2.0:1788878873.081747:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878873.081758:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.081774:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788878873.081783:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878873.081788:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878873.081792:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878873.081796:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1788878873.081800:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878873.081806:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878873.081809:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:1.0:1788878873.081817:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878873/real 1788878873] req@ffff94e5045e5880 x1875765290106752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788878873.081818:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878873.081827:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878873.081836:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878873.081851:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878873.081855:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788878873.081859:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788878873.081859:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878873.081868:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:1.0:1788878873.081870:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1788878873.081886:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.081889:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878873.081895:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878873.081899:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.081900:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.081967:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878873.081971:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878873.081977:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290107008/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788878873.081985:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290106752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878873.082030:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878873.082041:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878873.082043:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082050:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878873.082050:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082064:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878873.082064:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788878873.082076:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1788878873.082079:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788878873.082081:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082092:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878873.082092:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788878873.082101:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878873.082102:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788878873.082107:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788878873.082117:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788878873.082119:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878873.082129:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:2.0:1788878873.082134:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9811 00000100:00100000:1.0:1788878873.082139:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290106752:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878873.082149:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878873.082153:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878873.082154:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878873.082161:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878873.082166:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788878873.082166:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878873.082176:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788878873.082177:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878873.082181:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878873.082184:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000020:00000001:2.0:1788878873.082195:0:136860:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1788878873.082196:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878873.082209:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:2.0:1788878873.082210:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878873.082218:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788878873.082221:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000020:00000001:2.0:1788878873.082224:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788878873.082229:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878873.082230:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878873.082233:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878873.082242:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878873.082246:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878873.082253:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878873.082255:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878873.082258:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1788878873.082262:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878873.082266:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878873.082268:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878873.082278:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878873.082283:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878873.082284:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878873.082289:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00000001:2.0:1788878873.082293:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082295:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878873.082303:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082305:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878873.082316:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878873.082320:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788878873.082322:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878873.082325:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878873.082327:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878873.082330:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788878873.082333:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878873.082334:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878873.082343:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878873.082348:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290106880:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878873.082354:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788878873.082357:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878873.082358:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878873.082367:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878873.082370:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878873.082373:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878873.082401:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290107136/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878873.082402:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878873.082407:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 00000100:00000001:1.0:1788878873.082419:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878873.082433:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878873.082437:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878873.082444:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788878873.082460:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878873.082471:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878873.082480:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878873.082483:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082485:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878873.082488:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082493:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304484364 00000100:00000001:1.0:1788878873.082496:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304484905 00000100:00000001:1.0:1788878873.082498:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304484364 00000100:00000001:1.0:1788878873.082500:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304484905 00000100:00000001:1.0:1788878873.082502:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878873.082504:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082506:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878873.082511:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878873/real 1788878873] req@ffff94e5045e5180 x1875765290106880/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878873.082520:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878873.082521:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878873.082524:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878873.082525:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878873.082527:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082528:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878873.082530:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878873.082534:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290106880/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878873.082540:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082542:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878873.082544:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082545:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878873.082547:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082548:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878873.082560:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878873.082562:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878873.082565:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878873.082567:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878873.082570:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290106880:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878873.082573:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878873.082575:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878873.082577:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878873.082579:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878873.082582:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878873.082584:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878873.082587:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290107008:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878873.082590:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878873.082592:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878873.082594:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878873.082596:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878873.082599:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082604:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878873.082605:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:1.0:1788878873.082608:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878873.082620:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878873.082623:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878873.082626:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878873.082627:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082628:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878873.082630:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082633:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304484362 00000100:00000001:1.0:1788878873.082636:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304484905 00000100:00000001:1.0:1788878873.082637:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304484362 00000100:00000001:1.0:1788878873.082639:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304484905 00000100:00000001:1.0:1788878873.082641:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878873.082643:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082644:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878873.082648:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878873/real 1788878873] req@ffff94e5045e4a80 x1875765290107008/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878873.082654:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878873.082655:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878873.082657:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878873.082658:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878873.082659:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082660:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878873.082661:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878873.082664:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290107008/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878873.082670:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082672:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878873.082674:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082676:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878873.082677:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082678:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878873.082687:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878873.082689:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878873.082691:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878873.082694:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878873.082697:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290107008:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878873.082700:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082702:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878873.082703:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878873.082704:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878873.082706:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878873.082707:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878873.082708:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878873.082710:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878873.082711:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878873.082713:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878873.082714:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082716:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878873.082717:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878873.082718:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878873.082719:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878873.082720:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878873.082721:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878873.082723:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878873.082724:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878873.082725:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878873.082726:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082727:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082730:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878873.082732:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082733:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878873.082735:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878873.082736:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878873.082737:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878873.082739:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878873.082741:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878873.082742:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878873.082744:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878873.082746:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290107136:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878873.082749:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878873.082751:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878873.082752:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878873.082754:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878873.082756:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082776:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878873.082779:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:1.0:1788878873.082784:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878873.082800:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878873.082806:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878873.082809:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878873.082811:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082814:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878873.082817:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082822:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304484362 00000100:00000001:1.0:1788878873.082826:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304484905 00000100:00000001:1.0:1788878873.082829:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304484362 00000100:00000001:1.0:1788878873.082832:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304484905 00000100:00000001:1.0:1788878873.082835:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878873.082838:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082840:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878873.082846:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878873/real 1788878873] req@ffff94e5045e7100 x1875765290107136/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878873.082856:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878873.082857:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878873.082860:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878873.082862:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878873.082863:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082865:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878873.082866:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878873.082870:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290107136/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878873.082876:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082878:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878873.082880:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082882:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878873.082883:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082884:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878873.082943:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878873.082947:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878873.082951:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878873.082953:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878873.082956:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290107136:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878873.082960:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082963:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878873.082966:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878873.082968:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878873.082972:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878873.082974:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878873.082977:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878873.082979:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878873.082980:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878873.082982:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878873.082983:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082985:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.082988:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878873.082989:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.082991:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878873.082992:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878873.144959:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878873.144965:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878873.144967:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878873.144971:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878873.144974:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878873.144977:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.144978:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878873.144980:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878873.144982:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878873.849013:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878873.849019:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878873.849022:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878873.849023:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878873.849062:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878873.849064:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878873.849067:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878874.169008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878874.169015:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878874.169018:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878874.169023:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878874.169028:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878874.169031:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878874.169035:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878874.169037:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878874.169040:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878874.873070:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878874.873083:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878874.873087:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878874.873091:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878874.873098:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878874.873101:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878874.873103:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878874.873106:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878874.873110:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878874.873135:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878874.873220:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878874.873226:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878874.873231:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878875.192971:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878875.192978:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878875.192980:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878875.192986:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878875.192990:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878875.193004:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878875.193007:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878875.193010:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878875.193012:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878875.897098:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878875.897106:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878875.897114:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878875.897116:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878875.897200:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878875.897222:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878875.897227:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878876.216973:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878876.216979:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878876.216981:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878876.216993:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878876.216996:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878876.216999:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878876.217002:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878876.217004:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878876.217006:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878876.921050:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878876.921058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878876.921063:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878876.921064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878876.921105:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878876.921108:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878876.921111:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878877.241007:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878877.241014:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878877.241016:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878877.241022:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878877.241025:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878877.241029:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878877.241032:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878877.241035:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878877.241037:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878877.945051:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878877.945059:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878877.945065:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878877.945067:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878877.945094:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878877.945097:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878877.945100:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878878.201054:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878878.201069:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878878.201072:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878878.201077:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878878.201082:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878878.201090:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9817 00000100:00080000:2.0:1788878878.201113:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878878.201118:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878878.201120:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878878.201130:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878878.201133:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878878.201137:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878878.201140:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878878.201143:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878878.201145:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878878.201150:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878878.201153:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878878.201156:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878878.201160:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878878.201164:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878878.201166:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878878.201169:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878878.201182:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878878.201185:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878878.201190:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878878.201195:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878878.201202:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878878.201220:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878878.201235:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290107264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878878.201255:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878878.201263:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878878.201286:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878878.201291:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878878.201295:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878878.201297:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878878.201299:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878878.201302:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878878.201306:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9817 00000100:00080000:2.0:1788878878.201312:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878878.201315:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878878.201317:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878878.201322:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878878.201324:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878878.201328:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878878.201331:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878878.201334:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878878.201337:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878878.201341:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878878.201344:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878878.201345:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878878.201353:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878878.201356:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788878878.201361:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878878.201363:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878878.201372:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788878878.201374:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.201380:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878878.201382:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878878.201392:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788878878.201394:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878878.201403:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1788878878.201404:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878878.201417:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878878.201417:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878878.201447:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290107264:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878878.201447:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878878.201454:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.201460:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788878878.201461:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878878.201467:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788878878.201470:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878878.201471:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878878.201476:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878878.201481:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788878878.201484:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878878.201490:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878878.201498:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290107392/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878878.201511:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878878.201515:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878878.201516:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.201523:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000100:00000001:2.0:1788878878.201527:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.201530:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788878878.201534:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878878.201540:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878878.201543:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878878.201545:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878878.201549:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878878.201554:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9817 00000100:00080000:2.0:1788878878.201562:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878878.201567:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878878.201569:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878878.201577:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878878.201580:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878878.201584:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878878.201592:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000800:00000100:1.0:1788878878.201593:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878878.201603:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878878.201608:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878878.201611:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878878.201619:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878878.201622:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.201624:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878878.201630:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.201636:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304490025 00000100:00080000:2.0:1788878878.201638:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878878.201646:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878878.201649:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878878.201652:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1788878878.201655:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878878.201662:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.201664:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304490025 00000100:00000001:2.0:1788878878.201665:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878878.201672:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878878.201675:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788878878.201680:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1788878878.201687:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.201688:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878878.201694:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.201697:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1788878878.201705:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1788878878.201710:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878878/real 1788878878] req@ffff94e5045e7800 x1875765290107264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788878878.201710:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878878.201716:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878878.201720:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878878.201724:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878878.201765:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788878878.201790:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:2.0:1788878878.201798:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290107520/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1788878878.201800:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878878.201810:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878878.201818:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878878.201821:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878878.201823:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.201829:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878878.201831:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878878.201842:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788878878.201843:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1788878878.201849:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788878878.201852:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878878.201855:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878878.201860:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788878878.201868:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290107264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878878.201869:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878878.201880:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9817 00000100:00080000:2.0:1788878878.201891:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878878.201895:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878878.201897:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878878.201902:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878878.201903:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878878.201915:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788878878.201916:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788878878.201968:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1788878878.201972:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788878878.201974:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788878878.201982:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788878878.201985:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878878.201992:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1788878878.201995:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00080000:1.0:1788878878.202005:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788878878.202006:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878878.202014:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788878878.202024:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00080000:2.0:1788878878.202055:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878878.202059:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788878878.202066:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878878.202075:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1788878878.202076:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290107264:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788878878.202080:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878878.202091:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.202106:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878878.202107:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878878.202113:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878878.202115:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00000001:2.0:1788878878.202120:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.202121:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788878878.202126:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878878.202128:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878878.202136:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788878878.202138:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:2.0:1788878878.202141:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878878.202145:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:2.0:1788878878.202146:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878878.202153:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:1.0:1788878878.202155:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878878.202157:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878878.202167:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1788878878.202168:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878878.202175:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00100000:2.0:1788878878.202185:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290107648/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878878.202186:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878878.202192:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.202201:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.202218:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878878.202218:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878878.202225:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878878.202229:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.202230:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878878.202235:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878878.202239:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878878.202243:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878878.202244:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.202245:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878878.202258:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878878.202261:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878878.202265:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878878.202272:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290107392:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878878.202276:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878878.202280:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878878.202282:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878878.202284:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878878.202289:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.202299:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878878.202301:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 00000100:00000001:1.0:1788878878.202307:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878878.202327:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878878.202333:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878878.202336:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878878.202338:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.202340:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878878.202343:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.202349:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304489484 00000100:00000001:1.0:1788878878.202353:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304490025 00000100:00000001:1.0:1788878878.202358:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304489484 00000100:00000001:1.0:1788878878.202361:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304490025 00000100:00000001:1.0:1788878878.202365:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878878.202369:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.202372:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878878.202380:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878878/real 1788878878] req@ffff94e5045e6300 x1875765290107392/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878878.202398:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878878.202401:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878878.202406:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878878.202409:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878878.202411:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.202427:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878878.202429:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878878.202438:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290107392/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878878.202452:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.202457:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878878.202460:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.202463:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878878.202464:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.202467:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878878.202480:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878878.202483:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878878.202487:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878878.202490:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878878.202494:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290107392:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878878.202499:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878878.202501:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878878.202503:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878878.202506:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878878.202508:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878878.202511:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878878.202514:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290107520:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878878.202518:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878878.202520:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878878.202522:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878878.202524:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878878.202528:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.202536:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878878.202537:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:1.0:1788878878.202542:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878878.202558:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878878.202588:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878878.202592:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878878.202594:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.202596:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878878.202600:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.202605:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304489482 00000100:00000001:1.0:1788878878.202608:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304490025 00000100:00000001:1.0:1788878878.202611:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304489482 00000100:00000001:1.0:1788878878.202614:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304490025 00000100:00000001:1.0:1788878878.202616:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878878.202619:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.202621:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878878.202626:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878878/real 1788878878] req@ffff94e5045e5500 x1875765290107520/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878878.202636:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878878.202638:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878878.202641:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878878.202643:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878878.202644:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.202647:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878878.202649:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878878.202653:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290107520/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878878.202662:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.202665:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878878.202668:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.202670:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878878.202672:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.202674:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878878.202685:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878878.202689:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878878.202694:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878878.202698:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878878.202706:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290107520:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878878.202712:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.202716:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878878.202718:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878878.202722:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878878.202751:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878878.202781:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878878.202784:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878878.202788:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878878.202791:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878878.202795:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878878.202797:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.202801:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878878.202804:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878878.202807:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878878.202810:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878878.202812:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878878.202814:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878878.202818:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878878.202828:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878878.202848:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878878.202851:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.202855:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.202862:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878878.202864:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.202867:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878878.202869:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878878.202871:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878878.202873:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878878.202875:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878878.202879:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878878.202881:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878878.202895:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878878.202900:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290107648:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878878.202905:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878878.202908:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878878.202910:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878878.202912:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878878.202917:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.202925:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878878.202927:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:1.0:1788878878.202932:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878878.202985:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878878.202995:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878878.203000:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878878.203002:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.203006:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878878.203011:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.203018:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304489483 00000100:00000001:1.0:1788878878.203023:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304490025 00000100:00000001:1.0:1788878878.203027:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304489483 00000100:00000001:1.0:1788878878.203032:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304490025 00000100:00000001:1.0:1788878878.203037:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878878.203041:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.203045:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878878.203070:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878878/real 1788878878] req@ffff94e5045e5c00 x1875765290107648/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878878.203087:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878878.203089:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878878.203094:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878878.203097:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878878.203099:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.203101:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878878.203103:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878878.203108:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290107648/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878878.203116:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.203119:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878878.203122:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.203124:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878878.203126:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.203128:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878878.203143:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878878.203149:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878878.203154:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878878.203170:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878878.203177:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290107648:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878878.203184:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.203188:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878878.203190:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878878.203193:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878878.203198:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878878.203201:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878878.203203:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878878.203207:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878878.203210:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878878.203213:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878878.203215:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.203219:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.203226:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878878.203228:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.203230:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878878.203232:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878878.265011:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878878.265020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878878.265023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878878.265029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878878.265033:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878878.265038:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.265041:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878878.265044:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878878.265048:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878878.969045:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878878.969054:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878878.969061:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878878.969063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878878.969101:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878878.969105:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878878.969108:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878879.289033:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878879.289042:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878879.289044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878879.289050:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878879.289055:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878879.289060:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878879.289063:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878879.289066:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878879.289070:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878879.993015:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878879.993021:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878879.993026:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878879.993027:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878879.993088:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878879.993092:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878879.993096:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878880.312997:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878880.313005:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878880.313008:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878880.313016:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878880.313021:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878880.313026:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878880.313030:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878880.313033:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878880.313037:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878881.017043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878881.017051:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878881.017058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878881.017060:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878881.017086:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878881.017089:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878881.017093:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878881.337046:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878881.337057:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878881.337061:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878881.337070:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878881.337076:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878881.337083:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878881.337087:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878881.337090:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878881.337094:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878882.041022:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878882.041029:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878882.041035:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878882.041039:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878882.041041:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878882.041044:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878882.041045:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878882.041047:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878882.041049:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878882.041051:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878882.041093:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878882.041095:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878882.041098:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878882.360982:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878882.360988:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878882.360989:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878882.360993:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878882.360997:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878882.361001:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878882.361003:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878882.361007:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878882.361009:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878883.065049:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878883.065055:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878883.065061:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878883.065063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878883.065083:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878883.065102:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878883.065106:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878883.321106:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878883.321118:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878883.321121:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878883.321125:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878883.321130:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878883.321138:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9822 00000100:00080000:3.0:1788878883.321151:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878883.321154:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878883.321156:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878883.321165:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878883.321168:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878883.321172:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878883.321175:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878883.321178:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878883.321180:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878883.321184:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878883.321187:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878883.321190:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878883.321194:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878883.321197:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321200:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878883.321202:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878883.321215:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878883.321217:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321220:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878883.321223:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878883.321229:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321242:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878883.321252:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290107776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878883.321265:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878883.321270:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321289:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878883.321293:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878883.321297:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878883.321299:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878883.321301:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878883.321304:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878883.321307:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9822 00000100:00080000:3.0:1788878883.321310:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878883.321312:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878883.321313:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878883.321316:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878883.321318:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878883.321320:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878883.321322:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878883.321324:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878883.321326:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878883.321329:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878883.321332:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878883.321333:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878883.321339:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878883.321341:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321343:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878883.321347:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878883.321351:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321353:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878883.321357:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878883.321360:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321362:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878883.321364:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878883.321366:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321371:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878883.321377:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290107904/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878883.321383:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878883.321390:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878883.321396:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878883.321401:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878883.321403:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878883.321405:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321408:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878883.321415:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878883.321417:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878883.321424:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788878883.321424:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788878883.321436:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878883.321436:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878883.321442:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878883.321445:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788878883.321451:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290107776:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878883.321452:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878883.321461:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9822 00000100:00000001:1.0:1788878883.321466:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788878883.321468:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788878883.321473:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878883.321474:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788878883.321482:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321483:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788878883.321492:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1788878883.321493:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878883.321499:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878883.321502:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 02000000:00000001:1.0:1788878883.321504:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878883.321505:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878883.321513:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878883.321515:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878883.321518:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878883.321521:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878883.321523:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878883.321527:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878883.321529:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788878883.321530:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.321537:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:3.0:1788878883.321538:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878883.321543:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878883.321546:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:3.0:1788878883.321548:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321553:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878883.321559:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878883.321560:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321563:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878883.321565:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878883.321567:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321572:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878883.321577:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290108032/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878883.321585:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878883.321587:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321590:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878883.321593:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878883.321596:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878883.321598:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878883.321600:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878883.321603:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878883.321606:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9822 00000100:00080000:3.0:1788878883.321610:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878883.321613:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878883.321614:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321615:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878883.321626:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788878883.321627:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304495145 00000020:00000001:3.0:1788878883.321628:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878883.321632:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878883.321636:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878883.321639:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878883.321642:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878883.321643:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788878883.321648:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878883.321654:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.321657:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304495145 02000000:00000001:3.0:1788878883.321658:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878883.321665:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878883.321668:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1788878883.321670:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321676:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788878883.321678:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1788878883.321679:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878883.321710:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788878883.321714:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321720:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.321721:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878883.321726:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788878883.321730:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878883.321731:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321741:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878883.321743:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878883.321746:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.321748:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788878883.321751:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.321753:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788878883.321760:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1788878883.321761:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290108160/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878883.321765:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878883.321770:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878883.321775:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878883.321780:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321783:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788878883.321786:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878883.321792:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878883.321793:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878883.321802:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878883.321806:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878883.321829:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290107904:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878883.321837:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878883.321841:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878883.321845:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878883.321850:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878883.321858:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.321871:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878883.321876:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:1.0:1788878883.321885:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788878883.321914:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.321988:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304494604 00000100:00000001:1.0:1788878883.321994:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304495145 00000100:00000001:1.0:1788878883.321999:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304494604 00000100:00000001:1.0:1788878883.322002:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304495145 00000100:00000001:1.0:1788878883.322006:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878883.322009:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.322013:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878883.322016:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878883.322018:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878883.322021:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878883.322024:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878883.322027:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878883.322033:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290108032:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878883.322038:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878883.322041:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878883.322043:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878883.322045:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878883.322050:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.322058:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878883.322060:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:1.0:1788878883.322065:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788878883.322081:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.322085:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304494602 00000100:00000001:1.0:1788878883.322088:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304495145 00000100:00000001:1.0:1788878883.322091:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304494602 00000100:00000001:1.0:1788878883.322094:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304495145 00000100:00000001:1.0:1788878883.322096:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878883.322098:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.322101:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.322103:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878883.322108:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878883.322110:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788878883.322112:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878883.322114:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878883.322116:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878883.322118:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878883.322120:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878883.322123:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878883.322125:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878883.322127:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878883.322131:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290108160:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878883.322134:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878883.322136:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878883.322137:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878883.322139:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878883.322144:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.322149:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878883.322150:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:1.0:1788878883.322154:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788878883.322163:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.322166:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304494602 00000100:00000001:1.0:1788878883.322169:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304495145 00000100:00000001:1.0:1788878883.322172:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304494602 00000100:00000001:1.0:1788878883.322175:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304495145 00000100:00000001:1.0:1788878883.322177:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878883.322179:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.322182:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.322184:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878883.322187:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878883.322188:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788878883.322190:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878883.322192:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878883.322194:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878883.322196:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878883.384994:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878883.385001:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878883.385004:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878883.385008:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878883.385012:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878883.385016:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878883.385020:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878883.385023:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878883.385026:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878884.089081:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878884.089091:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878884.089095:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878884.089099:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878884.089106:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878884.089108:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878884.089112:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878884.089114:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878884.089130:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878884.089131:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878884.089221:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878884.089226:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878884.089231:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878884.409071:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878884.409083:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878884.409087:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878884.409096:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878884.409103:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878884.409109:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878884.409114:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878884.409118:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878884.409123:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878885.113117:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878885.113128:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878885.113135:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878885.113137:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878885.113277:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878885.113285:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878885.113294:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878885.433080:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878885.433111:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878885.433115:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878885.433123:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878885.433129:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878885.433134:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878885.433138:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878885.433142:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878885.433146:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878886.137068:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878886.137075:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878886.137082:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878886.137083:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878886.137107:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878886.137110:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878886.137113:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788878886.393166:0:2351:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1788878886.393202:0:2351:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788878886.393216:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788878886.393227:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878886.393243:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878886.393245:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878886.393247:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788878886.393254:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788878886.393257:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878886.393259:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878886.393260:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878886.393261:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788878886.393264:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788878886.393267:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878886.393269:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878886.393270:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878886.393271:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788878886.393274:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788878886.393278:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878886.393280:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878886.393281:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878886.393282:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878886.393294:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878886.393300:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878886.393303:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878886.393315:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878883/real 1788878886] req@ffff94e50449fb80 x1875765290108160/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878886.393339:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878886.393340:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878886.393346:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878886.393350:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878886.393352:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393355:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878886.393357:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878886.393362:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290108160/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878886.393371:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393374:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878886.393377:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393379:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878886.393380:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393382:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878886.393398:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878886.393402:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878886.393407:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878886.393411:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878886.393415:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290108160:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878886.393420:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878886.393438:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878883/real 1788878886] req@ffff94e50449d880 x1875765290107904/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878886.393449:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878886.393451:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878886.393455:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878886.393457:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878886.393458:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393460:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878886.393461:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878886.393466:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290107904/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878886.393472:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393474:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878886.393477:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393478:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878886.393479:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393481:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878886.393494:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878886.393497:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878886.393501:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878886.393503:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878886.393507:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290107904:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878886.393511:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878886.393515:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878883/real 1788878886] req@ffff94e50449c380 x1875765290108032/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878886.393524:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878886.393525:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878886.393528:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878886.393530:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878886.393531:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393533:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878886.393535:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878886.393539:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290108032/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878886.393547:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393549:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878886.393552:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393554:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878886.393555:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393556:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878886.393570:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878886.393573:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878886.393577:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878886.393579:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878886.393582:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290108032:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878886.393587:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878886.393591:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878883/real 1788878886] req@ffff94e50449ce00 x1875765290107776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878886.393601:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878886.393602:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878886.393605:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878886.393607:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878886.393608:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393610:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878886.393612:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878886.393616:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290107776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878886.393623:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393625:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878886.393627:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393629:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878886.393630:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393632:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878886.393634:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878886.393637:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878886.393641:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878886.393643:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878886.393646:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290107776:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878886.393651:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393653:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878886.393655:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878886.393658:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878886.393663:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878886.393667:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878886.393682:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878886.393687:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878886.393689:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878886.393693:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878886.393694:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393696:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878886.393697:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878886.393698:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878886.393700:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878886.393701:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878886.393702:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878886.393704:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878886.393705:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878886.393707:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878886.393708:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393709:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878886.393710:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878886.393711:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878886.393713:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878886.393714:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878886.393715:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878886.393717:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878886.393718:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878886.393719:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878886.393721:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393722:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878886.393723:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878886.393724:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878886.393726:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878886.393727:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878886.393728:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878886.393730:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878886.393731:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878886.393733:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878886.393734:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393736:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878886.393744:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878886.393746:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878886.393749:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878886.393750:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878886.457014:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878886.457021:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878886.457023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878886.457027:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878886.457031:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878886.457034:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878886.457036:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878886.457038:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878886.457041:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878887.161062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878887.161068:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878887.161073:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878887.161074:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878887.161108:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878887.161111:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878887.161113:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878887.480988:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878887.480995:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878887.480996:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878887.481000:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878887.481003:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878887.481006:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878887.481008:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878887.481011:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878887.481012:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878888.185010:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878888.185016:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878888.185020:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878888.185022:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878888.185095:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878888.185099:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878888.185102:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878888.441036:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878888.441046:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878888.441065:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878888.441068:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878888.441073:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878888.441080:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9827 00000100:00080000:3.0:1788878888.441089:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878888.441092:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878888.441094:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878888.441101:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878888.441103:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878888.441107:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878888.441109:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878888.441111:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878888.441129:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878888.441134:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878888.441137:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878888.441139:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878888.441143:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878888.441145:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878888.441148:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878888.441150:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878888.441159:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878888.441160:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878888.441163:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878888.441166:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878888.441171:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878888.441182:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878888.441191:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290108288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878888.441201:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878888.441205:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878888.441222:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878888.441225:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878888.441228:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878888.441230:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878888.441231:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878888.441234:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878888.441236:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9827 00000100:00080000:3.0:1788878888.441239:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878888.441241:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878888.441243:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878888.441245:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878888.441247:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878888.441249:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878888.441251:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878888.441254:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878888.441255:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878888.441259:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878888.441262:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878888.441263:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878888.441266:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788878888.441269:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878888.441288:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788878888.441289:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.441293:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878888.441294:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788878888.441299:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1788878888.441300:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878888.441303:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788878888.441305:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878888.441308:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878888.441309:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878888.441313:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878888.441313:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878888.441317:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878888.441318:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878888.441320:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878888.441323:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878888.441326:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878888.441327:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290108288:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878888.441334:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.441350:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878888.441357:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1788878888.441360:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290108416/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788878888.441361:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878888.441368:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878888.441375:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788878888.441377:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878888.441378:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878888.441387:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878888.441390:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878888.441393:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878888.441394:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878888.441395:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878888.441398:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878888.441400:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9827 00000100:00000001:1.0:1788878888.441419:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788878888.441422:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878888.441428:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000100:00000001:3.0:1788878888.441429:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878888.441432:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878888.441441:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788878888.441443:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1788878888.441444:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878888.441452:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878888.441456:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878888.441459:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878888.441460:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878888.441465:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878888.441483:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878888.441485:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878888.441488:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878888.441489:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878888.441491:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878888.441493:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878888.441494:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878888.441497:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878888.441504:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878888.441506:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878888.441508:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878888.441511:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000800:00000100:1.0:1788878888.441514:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788878888.441516:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878888.441526:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.441531:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00100000:3.0:1788878888.441537:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290108544/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878888.441539:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878888.441543:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.441548:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878888.441556:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878888.441557:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878888.441562:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.441566:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304500265 00000100:00000001:3.0:1788878888.441568:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878888.441575:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878888.441580:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788878888.441582:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878888.441583:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878888.441586:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878888.441593:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304500265 00000100:00000001:3.0:1788878888.441593:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878888.441598:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9827 00000100:00000001:1.0:1788878888.441603:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788878888.441607:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878888.441609:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878888.441612:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878888.441617:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878888.441618:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878888.441623:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788878888.441625:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1788878888.441628:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878888.441632:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878888.441637:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:1.0:1788878888.441638:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878888/real 1788878888] req@ffff94e50449e680 x1875765290108288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878888.441641:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878888.441644:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878888.441653:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878888.441678:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878888.441683:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878888.441690:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:1.0:1788878888.441691:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1788878888.441694:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788878888.441696:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878888.441699:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878888.441704:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878888.441705:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878888.441710:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000008:00000001:3.0:1788878888.441711:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.441715:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878888.441716:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.441723:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788878888.441726:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788878888.441728:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878888.441731:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878888.441734:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878888.441736:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878888.441738:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878888.441744:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878888.441751:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290108288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788878888.441756:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290108672/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878888.441780:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878888.441783:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878888.441788:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878888.441789:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878888.441795:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.441797:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.441802:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878888.441804:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.441808:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878888.441811:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878888.441816:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878888.441823:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878888.441826:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878888.441832:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290108288:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878888.441840:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.441843:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878888.441845:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878888.441848:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878888.441853:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878888.441855:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878888.441858:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878888.441862:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878888.441865:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878888.441869:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878888.441871:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.441874:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.441881:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878888.441884:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.441887:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878888.441889:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878888.441891:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878888.441892:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878888.441894:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878888.441897:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878888.441899:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878888.441934:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878888.441938:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290108416:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878888.441943:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878888.441945:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878888.441947:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878888.441949:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878888.441965:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.441973:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878888.441975:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:1.0:1788878888.441980:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878888.441999:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878888.442006:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878888.442009:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878888.442011:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.442013:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878888.442017:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.442022:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304499724 00000100:00000001:1.0:1788878888.442026:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304500265 00000100:00000001:1.0:1788878888.442029:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304499724 00000100:00000001:1.0:1788878888.442032:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304500265 00000100:00000001:1.0:1788878888.442035:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878888.442038:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.442041:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878888.442047:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878888/real 1788878888] req@ffff94e50449df80 x1875765290108416/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878888.442059:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878888.442060:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878888.442064:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878888.442067:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878888.442068:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442071:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878888.442072:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878888.442077:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290108416/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878888.442086:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442089:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878888.442092:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442094:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878888.442095:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442098:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878888.442114:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878888.442118:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878888.442123:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878888.442126:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878888.442130:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290108416:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878888.442135:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878888.442137:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878888.442138:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878888.442142:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878888.442144:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878888.442147:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878888.442151:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290108544:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878888.442159:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878888.442162:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878888.442164:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878888.442177:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878888.442181:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.442188:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878888.442190:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:1.0:1788878888.442195:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878888.442211:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878888.442217:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878888.442220:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878888.442222:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.442224:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878888.442228:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.442244:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304499722 00000100:00000001:1.0:1788878888.442247:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304500265 00000100:00000001:1.0:1788878888.442250:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304499722 00000100:00000001:1.0:1788878888.442253:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304500265 00000100:00000001:1.0:1788878888.442256:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878888.442258:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.442261:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878888.442266:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878888/real 1788878888] req@ffff94e50449ce00 x1875765290108544/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878888.442278:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878888.442279:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878888.442283:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878888.442285:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878888.442287:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442290:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878888.442292:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878888.442297:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290108544/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878888.442316:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442319:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878888.442322:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442324:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878888.442326:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442328:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878888.442344:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878888.442348:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878888.442352:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878888.442355:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878888.442360:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290108544:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878888.442365:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878888.442367:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878888.442369:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878888.442372:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878888.442375:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878888.442378:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878888.442388:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290108672:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878888.442393:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878888.442395:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878888.442400:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878888.442402:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878888.442406:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.442413:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878888.442415:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:1.0:1788878888.442418:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878888.442435:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878888.442441:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878888.442444:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878888.442446:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.442449:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878888.442452:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.442456:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304499722 00000100:00000001:1.0:1788878888.442459:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304500265 00000100:00000001:1.0:1788878888.442462:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304499722 00000100:00000001:1.0:1788878888.442465:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304500265 00000100:00000001:1.0:1788878888.442468:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878888.442471:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.442473:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878888.442478:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878888/real 1788878888] req@ffff94e50449c380 x1875765290108672/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878888.442488:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878888.442490:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878888.442493:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878888.442495:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878888.442496:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442499:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878888.442500:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878888.442505:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290108672/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878888.442512:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442515:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878888.442518:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442519:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878888.442521:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442523:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878888.442538:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878888.442541:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878888.442545:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878888.442548:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878888.442552:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290108672:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878888.442557:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442559:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878888.442562:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878888.442564:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878888.442567:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878888.442569:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878888.442571:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878888.442574:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878888.442576:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878888.442578:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878888.442580:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442581:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878888.442583:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878888.442584:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878888.442586:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878888.442587:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878888.442589:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878888.442591:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878888.442592:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878888.442594:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878888.442596:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442598:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878888.442599:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878888.442600:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878888.442602:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878888.442604:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878888.442605:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878888.442607:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878888.442609:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878888.442611:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878888.442613:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442616:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878888.442621:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878888.442623:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878888.442626:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878888.442628:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878888.505057:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878888.505063:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878888.505065:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878888.505098:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878888.505101:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878888.505104:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878888.505106:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878888.505109:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878888.505111:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878889.209020:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878889.209025:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878889.209030:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878889.209031:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878889.209088:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878889.209092:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878889.209096:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878889.529029:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878889.529049:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878889.529051:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878889.529058:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878889.529063:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878889.529068:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878889.529071:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878889.529075:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878889.529078:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878890.233012:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878890.233019:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878890.233024:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878890.233025:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878890.233047:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878890.233050:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878890.233053:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878890.552999:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878890.553006:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878890.553008:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878890.553014:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878890.553018:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878890.553022:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878890.553025:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878890.553027:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878890.553030:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878891.257089:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878891.257100:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878891.257110:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878891.257114:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878891.257153:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878891.257158:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878891.257164:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878891.576996:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878891.577015:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878891.577019:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878891.577026:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878891.577032:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878891.577037:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878891.577052:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878891.577056:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878891.577060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878892.281063:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878892.281074:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878892.281083:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878892.281087:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878892.281090:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878892.281094:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878892.281098:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878892.281101:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878892.281103:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878892.281106:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878892.281201:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878892.281207:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878892.281212:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878892.601021:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878892.601030:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878892.601032:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878892.601038:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878892.601042:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878892.601047:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878892.601050:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878892.601053:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878892.601056:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878893.305002:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878893.305007:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878893.305019:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878893.305021:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878893.305065:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878893.305081:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878893.305084:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878893.561010:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878893.561026:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878893.561029:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878893.561034:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878893.561040:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878893.561050:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9832 00000100:00080000:3.0:1788878893.561062:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878893.561066:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878893.561068:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878893.561078:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878893.561082:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878893.561086:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878893.561089:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878893.561092:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878893.561094:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878893.561100:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878893.561103:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878893.561107:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878893.561111:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878893.561115:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878893.561118:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878893.561120:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878893.561131:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878893.561133:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878893.561137:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878893.561140:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878893.561146:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878893.561160:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878893.561172:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290108800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878893.561186:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878893.561192:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878893.561216:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878893.561220:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878893.561225:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878893.561227:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878893.561229:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878893.561233:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878893.561236:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9832 00000100:00080000:3.0:1788878893.561240:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878893.561242:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878893.561244:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878893.561248:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878893.561250:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878893.561253:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878893.561257:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878893.561260:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878893.561262:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878893.561268:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878893.561271:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878893.561288:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878893.561290:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788878893.561297:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878893.561298:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878893.561305:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788878893.561307:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.561311:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788878893.561313:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788878893.561318:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788878893.561321:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788878893.561327:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1788878893.561329:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878893.561336:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878893.561337:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878893.561348:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878893.561349:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878893.561354:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878893.561360:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290108800:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878893.561361:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878893.561367:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878893.561374:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.561375:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878893.561383:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878893.561387:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878893.561388:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878893.561399:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878893.561406:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878893.561407:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290108928/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878893.561425:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878893.561428:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.561432:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878893.561434:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.561440:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00080000:3.0:1788878893.561441:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878893.561453:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788878893.561456:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878893.561457:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878893.561465:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878893.561469:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878893.561473:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9832 00000100:00080000:3.0:1788878893.561478:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878893.561481:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878893.561483:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878893.561488:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878893.561490:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878893.561493:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878893.561496:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878893.561500:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878893.561502:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878893.561506:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878893.561510:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878893.561512:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878893.561517:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878893.561525:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788878893.561527:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878893.561529:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878893.561536:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878893.561539:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878893.561543:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.561551:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788878893.561551:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878893.561559:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.561563:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878893.561564:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878893.561571:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788878893.561572:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878893.561577:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.561579:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878893.561591:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.561592:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.561606:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304505385 00000100:00100000:3.0:1788878893.561609:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290109056/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878893.561653:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878893.561656:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.561658:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788878893.561661:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878893.561668:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788878893.561670:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304505385 00000100:00080000:3.0:1788878893.561678:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788878893.561682:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878893.561685:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788878893.561690:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788878893.561691:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878893.561699:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878893.561702:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788878893.561707:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788878893.561712:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9832 00000100:00080000:3.0:1788878893.561720:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788878893.561724:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878893/real 1788878893] req@ffff94e50449d880 x1875765290108800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878893.561725:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878893.561732:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878893.561745:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878893.561750:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878893.561755:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878893.561765:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878893.561775:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878893.561780:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:1.0:1788878893.561781:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878893.561788:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1788878893.561793:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878893.561797:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878893.561799:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.561805:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:3.0:1788878893.561806:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878893.561813:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878893.561815:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878893.561818:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878893.561825:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878893.561827:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878893.561833:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:1.0:1788878893.561837:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290108800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788878893.561838:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878893.561851:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878893.561859:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.561870:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878893.561872:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788878893.561876:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878893.561878:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878893.561889:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788878893.561890:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878893.561892:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788878893.561895:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788878893.561900:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.561901:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.561945:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878893.561948:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878893.561953:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878893.561963:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878893.561967:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788878893.561969:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878893.561974:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290108800:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878893.561985:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.561992:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1788878893.561994:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290109184/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878893.561996:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878893.562002:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878893.562012:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878893.562020:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:3.0:1788878893.562022:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000020:00000001:1.0:1788878893.562027:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788878893.562033:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878893.562039:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1788878893.562040:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878893.562050:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878893.562059:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878893.562063:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562069:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562082:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878893.562086:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.562091:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878893.562094:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878893.562097:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878893.562101:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878893.562104:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878893.562108:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878893.562112:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878893.562117:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878893.562124:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290108928:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878893.562129:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878893.562132:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878893.562135:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878893.562137:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878893.562142:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.562151:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878893.562153:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:1.0:1788878893.562158:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878893.562179:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878893.562185:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878893.562189:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878893.562191:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.562193:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878893.562197:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.562203:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304504844 00000100:00000001:1.0:1788878893.562208:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304505385 00000100:00000001:1.0:1788878893.562213:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304504844 00000100:00000001:1.0:1788878893.562218:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304505385 00000100:00000001:1.0:1788878893.562222:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878893.562227:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.562230:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878893.562240:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878893/real 1788878893] req@ffff94e50449fb80 x1875765290108928/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878893.562272:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878893.562275:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878893.562281:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878893.562284:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878893.562287:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562291:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878893.562294:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878893.562302:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290108928/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878893.562313:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562317:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878893.562320:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562322:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878893.562323:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562326:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878893.562340:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878893.562343:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878893.562347:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878893.562350:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878893.562354:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290108928:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878893.562359:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878893.562361:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878893.562363:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878893.562366:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878893.562369:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878893.562371:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878893.562375:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290109056:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878893.562379:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878893.562381:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878893.562384:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878893.562386:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878893.562391:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.562399:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878893.562400:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:1.0:1788878893.562406:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878893.562423:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878893.562429:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878893.562432:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878893.562434:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.562437:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878893.562442:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.562448:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304504842 00000100:00000001:1.0:1788878893.562453:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304505385 00000100:00000001:1.0:1788878893.562457:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304504842 00000100:00000001:1.0:1788878893.562461:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304505385 00000100:00000001:1.0:1788878893.562465:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878893.562470:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.562474:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878893.562483:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878893/real 1788878893] req@ffff94e50449d180 x1875765290109056/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878893.562499:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878893.562502:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878893.562508:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878893.562511:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878893.562514:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562518:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878893.562521:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878893.562529:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290109056/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878893.562543:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562546:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878893.562550:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562552:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878893.562553:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562556:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878893.562569:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878893.562573:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878893.562577:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878893.562580:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878893.562584:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290109056:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878893.562588:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878893.562590:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878893.562592:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878893.562595:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878893.562597:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878893.562600:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878893.562604:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290109184:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878893.562608:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878893.562611:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878893.562613:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878893.562615:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878893.562643:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.562653:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878893.562655:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:1.0:1788878893.562660:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878893.562678:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878893.562684:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878893.562687:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878893.562690:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.562692:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878893.562696:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.562700:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304504842 00000100:00000001:1.0:1788878893.562703:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304505385 00000100:00000001:1.0:1788878893.562707:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304504842 00000100:00000001:1.0:1788878893.562710:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304505385 00000100:00000001:1.0:1788878893.562713:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878893.562715:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.562718:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878893.562723:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878893/real 1788878893] req@ffff94e50449f800 x1875765290109184/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878893.562732:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878893.562734:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878893.562737:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878893.562739:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878893.562741:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562743:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878893.562745:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878893.562749:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290109184/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878893.562759:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562761:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878893.562764:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562766:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878893.562768:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562770:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878893.562783:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878893.562786:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878893.562789:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878893.562792:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878893.562795:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290109184:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878893.562800:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562802:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878893.562805:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878893.562807:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878893.562810:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878893.562812:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878893.562814:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878893.562817:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878893.562819:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878893.562822:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878893.562823:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562825:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878893.562827:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878893.562829:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878893.562831:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878893.562833:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878893.562834:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878893.562836:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878893.562838:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878893.562839:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878893.562841:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562843:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878893.562844:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878893.562845:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878893.562847:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878893.562849:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878893.562850:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878893.562852:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878893.562853:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878893.562855:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878893.562856:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562858:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878893.562863:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878893.562865:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878893.562867:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878893.562869:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878893.625082:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878893.625093:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878893.625096:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878893.625105:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878893.625110:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878893.625116:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878893.625120:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878893.625124:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878893.625130:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878894.329083:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878894.329096:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878894.329100:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878894.329105:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878894.329113:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878894.329116:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878894.329119:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878894.329122:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878894.329127:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878894.329129:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878894.329176:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878894.329180:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878894.329185:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878894.649053:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878894.649064:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878894.649067:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878894.649074:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878894.649079:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878894.649085:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878894.649089:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878894.649092:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878894.649096:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878895.353043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878895.353055:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878895.353062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878895.353064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878895.353098:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878895.353102:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878895.353106:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878895.673039:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878895.673049:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878895.673052:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878895.673059:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878895.673064:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878895.673070:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878895.673074:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878895.673078:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878895.673081:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878896.377101:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878896.377109:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878896.377113:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878896.377114:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878896.377185:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878896.377189:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878896.377192:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878896.697140:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878896.697154:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878896.697159:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878896.697168:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878896.697177:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878896.697185:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878896.697190:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878896.697193:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878896.697197:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878897.401032:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878897.401037:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878897.401042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878897.401043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878897.401118:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878897.401122:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878897.401126:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878897.720992:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878897.721000:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878897.721004:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878897.721010:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878897.721015:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878897.721019:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878897.721023:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878897.721026:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878897.721030:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878898.425091:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878898.425118:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878898.425126:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878898.425127:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878898.425161:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878898.425164:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878898.425168:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878898.681212:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878898.681222:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878898.681225:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878898.681229:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878898.681234:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878898.681241:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9837 00000100:00080000:2.0:1788878898.681264:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878898.681267:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878898.681269:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878898.681277:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878898.681280:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878898.681284:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878898.681287:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878898.681289:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878898.681291:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878898.681295:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878898.681298:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878898.681301:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878898.681304:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878898.681307:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681310:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878898.681312:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878898.681323:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878898.681325:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681328:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878898.681331:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878898.681336:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681362:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878898.681373:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4380 x1875765290109312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878898.681384:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878898.681389:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681420:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878898.681423:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878898.681428:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878898.681429:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878898.681431:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878898.681434:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878898.681436:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9837 00000100:00080000:2.0:1788878898.681440:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878898.681442:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878898.681443:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878898.681446:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878898.681447:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878898.681450:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878898.681463:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878898.681465:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878898.681466:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878898.681470:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878898.681472:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878898.681473:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878898.681480:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878898.681482:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681483:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878898.681487:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878898.681490:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681492:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878898.681496:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878898.681498:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681500:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878898.681501:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878898.681503:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681509:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878898.681515:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5f80 x1875765290109440/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878898.681536:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878898.681539:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681543:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878898.681546:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878898.681550:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878898.681552:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878898.681554:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878898.681556:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878898.681559:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9837 00000100:00080000:2.0:1788878898.681562:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878898.681565:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878898.681565:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878898.681574:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878898.681576:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788878898.681579:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788878898.681581:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788878898.681582:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788878898.681600:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878898.681602:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878898.681607:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878898.681612:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878898.681626:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878898.681628:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788878898.681630:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1788878898.681639:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878898.681644:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878898.681646:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788878898.681647:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878898.681650:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878898.681656:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878898.681658:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681664:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878898.681668:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878898.681673:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290109312:192.168.203.129@tcp:250:kworker.0 00000008:00000001:2.0:1788878898.681675:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681683:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.681689:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878898.681693:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878898.681696:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878898.681697:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788878898.681700:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878898.681703:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878898.681718:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878898.681721:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681723:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878898.681732:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1788878898.681740:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290109568/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788878898.681750:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681761:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878898.681763:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681770:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878898.681784:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878898.681787:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878898.681789:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878898.681791:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878898.681794:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788878898.681797:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788878898.681798:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9837 00000100:00000001:1.0:1788878898.681803:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074549120 : -117763635002496 : ffff94e5045e4380) 00000100:00080000:2.0:1788878898.681806:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878898.681810:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878898.681811:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878898.681814:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878898.681818:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878898.681820:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878898.681822:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878898.681824:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878898.681827:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878898.681830:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878898.681833:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878898.681836:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878898.681838:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878898.681843:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878898.681845:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681847:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878898.681849:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878898.681852:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681855:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878898.681860:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878898.681864:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788878898.681866:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878898.681872:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878898.681874:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878898.681878:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681885:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.681889:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00100000:2.0:1788878898.681895:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290109696/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878898.681898:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878898.681902:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.681923:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878898.681932:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.681940:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304510506 00000100:00000001:2.0:1788878898.681967:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878898.681972:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878898.681987:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.682025:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788878898.682036:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304510506 00000100:00000001:1.0:1788878898.682060:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788878898.682065:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878898.682070:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.682075:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878898.682088:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878898/real 1788878898] req@ffff94e5045e4380 x1875765290109312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878898.682106:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878898.682108:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878898.682114:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878898.682117:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878898.682120:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682124:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878898.682126:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878898.682134:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4380 x1875765290109312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878898.682144:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682149:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878898.682153:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682155:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878898.682156:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682176:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878898.682178:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878898.682181:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878898.682184:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878898.682187:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878898.682190:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290109312:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878898.682195:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878898.682196:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878898.682198:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878898.682200:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878898.682202:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878898.682203:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878898.682206:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290109440:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878898.682209:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878898.682211:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878898.682213:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878898.682214:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878898.682217:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.682223:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878898.682224:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074556288 : -117763634995328 : ffff94e5045e5f80) 00000100:00000001:1.0:1788878898.682245:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878898.682254:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878898.682257:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878898.682259:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878898.682260:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.682262:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878898.682264:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.682267:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304509964 00000100:00000001:1.0:1788878898.682269:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304510506 00000100:00000001:1.0:1788878898.682271:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304509964 00000100:00000001:1.0:1788878898.682273:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304510506 00000100:00000001:1.0:1788878898.682274:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878898.682276:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.682277:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878898.682279:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878898/real 1788878898] req@ffff94e5045e5f80 x1875765290109440/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878898.682284:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878898.682285:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878898.682287:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878898.682288:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878898.682289:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682290:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878898.682291:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878898.682293:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5f80 x1875765290109440/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878898.682306:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682310:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878898.682312:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682313:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878898.682314:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682315:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878898.682331:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878898.682332:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878898.682351:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878898.682353:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878898.682357:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290109440:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878898.682361:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682363:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878898.682364:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878898.682367:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878898.682371:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878898.682373:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878898.682375:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878898.682380:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878898.682382:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878898.682385:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878898.682386:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682387:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878898.682388:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878898.682389:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878898.682391:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878898.682392:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878898.682393:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878898.682394:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878898.682395:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878898.682396:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878898.682397:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682399:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682407:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878898.682409:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.682411:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878898.682412:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878898.682413:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878898.682414:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878898.682415:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878898.682417:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878898.682419:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878898.682420:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878898.682423:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290109568:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878898.682426:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878898.682428:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878898.682429:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878898.682430:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878898.682433:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.682458:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878898.682460:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:1.0:1788878898.682465:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878898.682480:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878898.682485:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878898.682488:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878898.682490:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.682492:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878898.682495:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.682521:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304509962 00000100:00000001:1.0:1788878898.682523:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304510506 00000100:00000001:1.0:1788878898.682525:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304509962 00000100:00000001:1.0:1788878898.682527:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304510506 00000100:00000001:1.0:1788878898.682528:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878898.682530:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.682531:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878898.682533:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878898/real 1788878898] req@ffff94e5045e5c00 x1875765290109568/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878898.682538:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878898.682539:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878898.682540:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878898.682541:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878898.682542:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682543:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878898.682544:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878898.682547:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290109568/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878898.682552:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682553:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878898.682554:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682555:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878898.682557:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682558:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878898.682571:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878898.682573:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878898.682575:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878898.682577:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878898.682579:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290109568:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878898.682581:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878898.682582:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878898.682594:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878898.682597:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878898.682598:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878898.682599:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878898.682601:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290109696:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878898.682604:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878898.682605:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878898.682607:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878898.682608:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878898.682626:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.682631:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878898.682632:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:1.0:1788878898.682635:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878898.682644:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878898.682647:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878898.682649:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878898.682651:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.682652:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878898.682654:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.682657:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304509962 00000100:00000001:1.0:1788878898.682659:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304510506 00000100:00000001:1.0:1788878898.682661:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304509962 00000100:00000001:1.0:1788878898.682663:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304510506 00000100:00000001:1.0:1788878898.682665:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878898.682667:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.682668:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878898.682671:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878898/real 1788878898] req@ffff94e5045e5500 x1875765290109696/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878898.682678:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878898.682679:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878898.682681:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878898.682682:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878898.682683:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682684:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878898.682685:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878898.682687:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290109696/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878898.682692:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682694:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878898.682695:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682696:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878898.682698:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682699:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878898.682712:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878898.682714:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878898.682717:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878898.682719:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878898.682722:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290109696:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878898.682724:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682725:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878898.682726:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878898.682727:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878898.682729:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878898.682730:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878898.682731:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878898.682733:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878898.682734:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878898.682736:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878898.682737:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682738:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878898.682739:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878898.682740:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878898.682741:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878898.682742:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878898.682742:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878898.682745:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878898.682746:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878898.682747:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878898.682748:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682765:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878898.682767:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878898.682768:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878898.682770:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878898.682771:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878898.745107:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878898.745116:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878898.745119:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878898.745127:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878898.745133:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878898.745138:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878898.745141:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878898.745144:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878898.745148:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878899.449043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878899.449052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878899.449057:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878899.449058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878899.449090:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878899.449094:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878899.449098:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878899.769023:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878899.769032:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878899.769034:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878899.769039:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878899.769043:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878899.769047:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878899.769050:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878899.769052:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878899.769055:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878900.473038:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878900.473057:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878900.473065:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878900.473066:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878900.473135:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878900.473141:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878900.473146:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878900.793073:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878900.793082:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878900.793085:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878900.793092:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878900.793097:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878900.793102:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878900.793105:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878900.793109:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878900.793113:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878901.497054:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878901.497061:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878901.497067:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878901.497068:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878901.497185:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878901.497197:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878901.497206:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878901.817060:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878901.817069:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878901.817073:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878901.817082:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878901.817088:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878901.817094:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878901.817099:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878901.817102:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878901.817107:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878902.521037:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878902.521049:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878902.521059:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878902.521065:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878902.521069:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878902.521076:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878902.521081:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878902.521083:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878902.521086:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878902.521090:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878902.521121:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878902.521127:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878902.521132:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878902.841077:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878902.841088:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878902.841090:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878902.841097:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878902.841102:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878902.841107:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878902.841110:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878902.841114:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878902.841117:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878903.545042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878903.545050:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878903.545056:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878903.545058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878903.545087:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878903.545090:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878903.545093:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878903.801094:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878903.801110:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878903.801114:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878903.801119:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878903.801125:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878903.801136:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9842 00000100:00080000:3.0:1788878903.801152:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878903.801156:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878903.801159:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878903.801170:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878903.801174:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878903.801179:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878903.801182:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878903.801185:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878903.801188:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878903.801194:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878903.801197:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878903.801201:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878903.801208:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878903.801213:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878903.801217:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878903.801222:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878903.801244:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878903.801248:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878903.801255:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878903.801260:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878903.801268:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878903.801291:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878903.801308:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290109824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878903.801323:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878903.801329:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878903.801353:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878903.801358:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878903.801362:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878903.801364:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878903.801367:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878903.801371:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878903.801374:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9842 00000100:00080000:3.0:1788878903.801380:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878903.801383:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878903.801385:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878903.801389:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878903.801391:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878903.801394:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878903.801397:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878903.801401:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878903.801403:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878903.801408:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878903.801412:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878903.801414:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878903.801422:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878903.801425:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878903.801428:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878903.801433:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878903.801439:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:3.0:1788878903.801439:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878903.801443:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.801452:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788878903.801453:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878903.801468:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.801471:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878903.801471:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878903.801479:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788878903.801481:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788878903.801484:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.801486:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878903.801496:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878903.801498:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878903.801501:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878903.801510:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878903.801519:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290109952/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788878903.801520:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290109824:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878903.801535:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878903.801543:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878903.801569:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878903.801574:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788878903.801576:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878903.801586:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878903.801588:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878903.801600:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878903.801608:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878903.801613:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878903.801615:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878903.801618:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878903.801623:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878903.801623:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788878903.801633:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00080000:3.0:1788878903.801634:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9842 00000100:00080000:3.0:1788878903.801645:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878903.801649:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878903.801653:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878903.801660:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878903.801669:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878903.801672:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878903.801677:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878903.801682:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878903.801686:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878903.801689:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878903.801695:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878903.801702:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878903.801706:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878903.801713:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878903.801718:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788878903.801728:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878903.801728:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878903.801737:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878903.801745:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878903.801749:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.801754:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788878903.801760:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788878903.801763:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788878903.801765:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.801770:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878903.801771:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878903.801779:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788878903.801780:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788878903.801784:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.801792:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878903.801796:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.801813:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304515625 00000100:00100000:3.0:1788878903.801819:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290110080/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878903.801831:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788878903.801844:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304515625 00000100:00000001:3.0:1788878903.801846:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878903.801854:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878903.801855:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.801869:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878903.801869:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.801882:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878903.801882:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788878903.801892:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788878903.801894:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878903.801899:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878903.801902:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788878903.801954:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878903/real 1788878903] req@ffff94e50449f480 x1875765290109824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878903.801965:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878903.801978:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9842 00000400:00000001:1.0:1788878903.801979:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878903.801984:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878903.801994:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:3.0:1788878903.801997:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878903.802000:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878903.802006:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878903.802009:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878903.802012:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878903.802017:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878903.802017:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:1.0:1788878903.802030:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290109824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788878903.802032:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878903.802041:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788878903.802050:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788878903.802050:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788878903.802059:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1788878903.802062:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788878903.802064:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878903.802071:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878903.802072:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878903.802078:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878903.802079:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878903.802083:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878903.802088:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788878903.802090:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00080000:1.0:1788878903.802095:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788878903.802097:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878903.802105:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:1.0:1788878903.802111:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 02000000:00000001:3.0:1788878903.802113:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878903.802122:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1788878903.802122:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878903.802128:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878903.802136:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878903.802140:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290109824:192.168.203.129@tcp:250:kworker.0 00000008:00000001:3.0:1788878903.802141:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878903.802148:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878903.802161:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788878903.802163:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878903.802165:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.802173:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878903.802174:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878903.802181:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788878903.802183:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788878903.802185:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878903.802193:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788878903.802197:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878903.802205:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:3.0:1788878903.802213:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290110208/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:1.0:1788878903.802215:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:3.0:1788878903.802233:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000020:00000001:1.0:1788878903.802234:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788878903.802246:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878903.802247:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878903.802254:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878903.802256:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.802260:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878903.802265:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.802270:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.802280:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878903.802283:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.802286:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878903.802289:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878903.802291:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878903.802293:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878903.802296:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878903.802300:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878903.802303:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878903.802306:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878903.802312:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290109952:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878903.802317:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878903.802320:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878903.802322:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878903.802325:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878903.802328:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.802338:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878903.802340:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:1.0:1788878903.802346:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878903.802364:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878903.802373:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878903.802377:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878903.802380:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.802383:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878903.802388:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.802397:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304515084 00000100:00000001:1.0:1788878903.802403:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304515625 00000100:00000001:1.0:1788878903.802407:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304515084 00000100:00000001:1.0:1788878903.802411:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304515625 00000100:00000001:1.0:1788878903.802415:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878903.802420:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.802423:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878903.802432:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878903/real 1788878903] req@ffff94e50449c700 x1875765290109952/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878903.802450:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878903.802453:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878903.802458:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878903.802467:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878903.802469:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.802472:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878903.802474:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878903.802479:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290109952/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878903.802488:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.802491:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878903.802494:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.802496:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878903.802497:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.802499:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878903.802518:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878903.802521:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878903.802525:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878903.802528:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878903.802533:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290109952:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878903.802537:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878903.802539:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878903.802541:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878903.802544:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878903.802546:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878903.802570:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878903.802575:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290110080:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878903.802579:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878903.802582:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878903.802584:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878903.802586:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878903.802590:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.802599:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878903.802601:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:1.0:1788878903.802607:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878903.802631:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878903.802639:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878903.802643:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878903.802646:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.802649:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878903.802654:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.802661:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304515082 00000100:00000001:1.0:1788878903.802666:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304515625 00000100:00000001:1.0:1788878903.802670:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304515082 00000100:00000001:1.0:1788878903.802675:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304515625 00000100:00000001:1.0:1788878903.802679:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878903.802683:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.802687:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878903.802695:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878903/real 1788878903] req@ffff94e50449e300 x1875765290110080/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878903.802716:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878903.802718:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878903.802722:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878903.802724:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878903.802726:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.802728:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878903.802730:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878903.802735:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290110080/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878903.802743:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.802746:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878903.802749:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.802751:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878903.802752:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.802754:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878903.802771:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878903.802774:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878903.802777:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878903.802779:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878903.802783:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290110080:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878903.802788:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878903.802790:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878903.802791:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878903.802794:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878903.802796:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878903.802798:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878903.802801:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290110208:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878903.802804:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878903.802807:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878903.802809:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878903.802811:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878903.802815:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.802821:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878903.802823:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:1.0:1788878903.802828:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878903.802843:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878903.802848:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878903.802850:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878903.802852:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.802854:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878903.802857:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.802861:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304515082 00000100:00000001:1.0:1788878903.802864:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304515625 00000100:00000001:1.0:1788878903.802867:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304515082 00000100:00000001:1.0:1788878903.802870:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304515625 00000100:00000001:1.0:1788878903.802872:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878903.802875:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.802876:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878903.802881:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878903/real 1788878903] req@ffff94e50449f100 x1875765290110208/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878903.802889:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878903.802890:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878903.802892:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878903.802894:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878903.802896:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.802898:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878903.802929:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878903.802935:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290110208/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878903.802943:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.802946:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878903.802950:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.802951:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878903.802953:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.802955:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878903.802972:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878903.802976:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878903.802979:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878903.802982:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878903.802986:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290110208:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878903.802989:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.802992:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878903.802993:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878903.802995:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878903.802998:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878903.803000:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878903.803002:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878903.803005:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878903.803006:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878903.803009:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878903.803011:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.803013:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878903.803014:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878903.803016:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878903.803018:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878903.803019:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878903.803020:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878903.803022:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878903.803023:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878903.803026:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878903.803027:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.803029:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878903.803030:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878903.803031:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878903.803033:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878903.803034:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878903.803035:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878903.803037:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878903.803039:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878903.803041:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878903.803042:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.803044:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878903.803049:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878903.803051:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878903.803054:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878903.803056:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878903.865120:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878903.865130:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878903.865133:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878903.865141:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878903.865146:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878903.865152:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878903.865156:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878903.865160:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878903.865163:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878904.569023:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878904.569034:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878904.569037:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878904.569041:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878904.569048:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878904.569051:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878904.569065:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878904.569068:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878904.569074:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878904.569075:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878904.569137:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878904.569144:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878904.569149:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878904.889107:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878904.889116:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878904.889119:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878904.889125:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878904.889130:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878904.889135:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878904.889139:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878904.889143:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878904.889146:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878905.593069:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878905.593090:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878905.593097:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878905.593099:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878905.593223:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878905.593229:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878905.593234:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878905.913014:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878905.913021:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878905.913023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878905.913028:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878905.913031:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878905.913035:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878905.913038:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878905.913041:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878905.913043:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878906.617061:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878906.617069:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878906.617074:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878906.617075:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878906.617103:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878906.617106:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878906.617109:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878906.937067:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878906.937075:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878906.937077:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878906.937082:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878906.937085:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878906.937100:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878906.937102:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878906.937105:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878906.937107:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878907.641044:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878907.641052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878907.641058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878907.641060:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878907.641087:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878907.641090:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878907.641094:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878907.961100:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878907.961109:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878907.961112:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878907.961118:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878907.961123:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878907.961128:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878907.961132:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878907.961135:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878907.961138:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878908.665000:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878908.665006:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878908.665011:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878908.665013:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878908.665114:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878908.665120:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878908.665124:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878908.920988:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878908.920997:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878908.920999:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878908.921003:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878908.921008:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878908.921015:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9847 00000100:00080000:3.0:1788878908.921026:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878908.921046:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878908.921048:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878908.921058:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878908.921061:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878908.921065:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878908.921068:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878908.921071:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878908.921074:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878908.921079:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878908.921082:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878908.921086:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878908.921090:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878908.921093:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878908.921096:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878908.921099:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878908.921109:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878908.921111:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878908.921115:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878908.921119:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878908.921124:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878908.921140:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878908.921152:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290110336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878908.921174:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878908.921180:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878908.921218:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878908.921221:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878908.921225:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878908.921227:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878908.921230:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878908.921233:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878908.921235:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9847 00000100:00080000:3.0:1788878908.921240:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878908.921242:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878908.921243:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878908.921247:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878908.921248:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878908.921250:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878908.921253:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878908.921272:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878908.921274:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878908.921279:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878908.921282:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878908.921283:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878908.921290:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878908.921293:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878908.921296:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878908.921301:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878908.921305:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878908.921308:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878908.921314:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878908.921315:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878908.921318:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878908.921321:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788878908.921322:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878908.921328:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.921331:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878908.921337:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.921338:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878908.921345:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878908.921351:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878908.921352:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290110464/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788878908.921361:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878908.921367:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878908.921373:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878908.921381:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290110336:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878908.921389:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878908.921396:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878908.921401:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878908.921405:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878908.921415:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.921439:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878908.921445:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:2.0:1788878908.921452:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878908.921480:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878908.921485:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878908.921509:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878908.921512:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878908.921532:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878908.921534:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000800:00000100:2.0:1788878908.921540:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1788878908.921549:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878908.921554:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878908.921557:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788878908.921562:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878908.921567:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.921571:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878908.921579:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.921584:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304520745 00000100:00000001:2.0:1788878908.921604:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878908.921612:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304520745 00000100:00000001:2.0:1788878908.921618:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878908.921624:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878908.921627:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.921631:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878908.921643:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878908/real 1788878908] req@ffff94e50449c000 x1875765290110336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878908.921656:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878908.921660:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878908.921668:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878908.921677:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878908.921681:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.921685:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878908.921689:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788878908.921697:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9847 00000100:00100000:2.0:1788878908.921698:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290110336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788878908.921711:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878908.921712:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878908.921719:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878908.921725:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878908.921731:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.921735:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878908.921737:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.921740:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878908.921743:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878908.921745:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878908.921750:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878908.921756:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878908.921761:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290110336:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878908.921767:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878908.921772:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.921776:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1788878908.921776:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878908.921781:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788878908.921782:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1788878908.921786:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878908.921790:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878908.921792:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1788878908.921795:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788878908.921795:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788878908.921801:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:3.0:1788878908.921803:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000020:00000001:2.0:1788878908.921806:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:3.0:1788878908.921809:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878908.921810:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788878908.921816:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878908.921819:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788878908.921822:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878908.921823:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:3.0:1788878908.921830:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.921832:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878908.921836:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878908.921837:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1788878908.921841:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878908.921843:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1788878908.921850:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878908.921854:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.921856:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878908.921859:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878908.921862:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878908.921864:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788878908.921866:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.921868:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878908.921871:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878908.921872:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788878908.921874:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878908.921876:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788878908.921878:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.921879:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878908.921887:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878908.921889:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878908.921890:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878908.921898:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878908.921936:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290110592/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788878908.921937:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290110464:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878908.921948:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878908.921953:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878908.921955:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788878908.921956:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878908.921960:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878908.921961:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878908.921967:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878908.921969:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878908.921975:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788878908.921978:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788878908.921979:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878908.921982:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:3.0:1788878908.921985:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788878908.921989:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788878908.921990:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878908.921996:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878908.922002:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9847 00000100:00080000:3.0:1788878908.922009:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878908.922013:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000800:00000100:2.0:1788878908.922018:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878908.922019:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878908.922026:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878908.922028:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788878908.922030:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:3.0:1788878908.922030:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878908.922035:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878908.922037:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878908.922055:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878908.922061:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878908.922061:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878908.922068:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:3.0:1788878908.922071:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878908.922075:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878908.922077:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878908.922083:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878908.922085:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304520204 02000000:00000001:3.0:1788878908.922088:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878908.922092:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304520745 02000000:00000001:3.0:1788878908.922094:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.922100:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304520204 00000100:00000001:3.0:1788878908.922101:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878908.922106:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878908.922108:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304520745 00000008:00000001:3.0:1788878908.922112:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.922115:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878908.922120:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.922121:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.922126:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1788878908.922130:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878908.922135:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878908.922137:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878908/real 1788878908] req@ffff94e50449ca80 x1875765290110464/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878908.922139:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878908.922144:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878908.922149:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788878908.922160:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788878908.922161:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788878908.922164:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878908.922170:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878908.922174:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00100000:3.0:1788878908.922175:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290110720/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878908.922176:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922184:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878908.922204:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878908.922216:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290110464/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878908.922216:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878908.922222:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878908.922228:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.922235:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922239:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878908.922241:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922243:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878908.922245:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922246:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878908.922280:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878908.922284:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878908.922288:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878908.922291:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878908.922295:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290110464:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878908.922300:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922312:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878908.922313:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878908.922315:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878908.922318:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878908.922319:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878908.922321:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878908.922323:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878908.922325:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878908.922327:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878908.922328:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922330:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922334:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878908.922336:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.922339:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878908.922340:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878908.922342:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878908.922344:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878908.922345:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878908.922348:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878908.922350:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878908.922352:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878908.922355:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290110592:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878908.922359:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878908.922361:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878908.922363:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878908.922365:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878908.922368:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.922375:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878908.922377:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:2.0:1788878908.922381:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878908.922397:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878908.922404:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878908.922408:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878908.922410:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.922414:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878908.922418:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.922423:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304520202 00000100:00000001:2.0:1788878908.922427:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304520745 00000100:00000001:2.0:1788878908.922431:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304520202 00000100:00000001:2.0:1788878908.922435:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304520745 00000100:00000001:2.0:1788878908.922439:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878908.922442:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.922445:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878908.922453:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878908/real 1788878908] req@ffff94e50449c000 x1875765290110592/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878908.922469:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878908.922472:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878908.922476:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878908.922479:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878908.922482:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922485:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878908.922487:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878908.922493:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290110592/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878908.922501:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922504:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878908.922507:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922508:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878908.922510:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922512:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878908.922546:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878908.922550:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878908.922570:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878908.922571:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878908.922582:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290110592:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878908.922585:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878908.922587:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878908.922588:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878908.922590:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878908.922592:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878908.922594:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878908.922596:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290110720:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878908.922599:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878908.922601:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878908.922602:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878908.922604:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878908.922607:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.922612:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878908.922613:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:2.0:1788878908.922616:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878908.922628:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878908.922632:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878908.922634:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878908.922646:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.922648:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878908.922650:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.922653:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304520202 00000100:00000001:2.0:1788878908.922656:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304520745 00000100:00000001:2.0:1788878908.922658:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304520202 00000100:00000001:2.0:1788878908.922660:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304520745 00000100:00000001:2.0:1788878908.922661:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878908.922663:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.922664:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878908.922668:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878908/real 1788878908] req@ffff94e50449f100 x1875765290110720/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878908.922680:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878908.922681:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878908.922683:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878908.922684:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878908.922686:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922687:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878908.922688:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878908.922691:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290110720/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878908.922696:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922699:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878908.922700:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922702:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878908.922703:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922705:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878908.922715:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878908.922717:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878908.922720:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878908.922721:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878908.922724:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290110720:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878908.922727:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922729:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878908.922730:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878908.922731:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878908.922733:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878908.922734:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878908.922735:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878908.922737:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878908.922738:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878908.922739:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878908.922740:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922742:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878908.922743:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878908.922744:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878908.922746:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878908.922747:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878908.922748:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878908.922749:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878908.922750:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878908.922751:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878908.922752:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922754:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878908.922757:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878908.922758:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878908.922759:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878908.922761:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878908.985144:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878908.985164:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878908.985168:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878908.985175:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878908.985179:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878908.985184:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878908.985187:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878908.985190:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878908.985194:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878909.688989:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878909.688995:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878909.688999:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878909.689000:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878909.689079:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878909.689083:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878909.689086:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878910.009001:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878910.009008:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878910.009011:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878910.009015:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878910.009019:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878910.009023:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878910.009025:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878910.009028:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878910.009031:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878910.712985:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878910.712990:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878910.712995:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878910.712996:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878910.713069:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878910.713071:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878910.713073:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878911.032973:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878911.032979:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878911.032980:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878911.032984:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878911.032987:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878911.032990:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878911.032992:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878911.032994:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878911.032997:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878911.737012:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878911.737032:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878911.737039:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878911.737042:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878911.737113:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878911.737118:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878911.737122:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878912.057010:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878912.057019:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878912.057021:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878912.057028:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878912.057032:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878912.057037:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878912.057040:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878912.057044:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878912.057047:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878912.761082:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878912.761092:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878912.761097:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878912.761099:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878912.761101:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878912.761104:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878912.761106:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878912.761107:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878912.761108:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878912.761110:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878912.761196:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878912.761214:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878912.761217:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878913.080985:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878913.080991:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878913.080992:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878913.080996:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878913.080999:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878913.081002:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878913.081004:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878913.081006:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878913.081008:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878913.785002:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878913.785008:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878913.785012:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878913.785013:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878913.785111:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878913.785116:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878913.785120:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878914.041002:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878914.041010:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878914.041012:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878914.041015:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878914.041018:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878914.041042:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9852 00000100:00080000:0.0:1788878914.041051:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878914.041053:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878914.041054:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878914.041061:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878914.041063:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878914.041066:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878914.041068:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878914.041070:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878914.041071:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878914.041073:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878914.041075:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878914.041077:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878914.041079:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878914.041081:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041083:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878914.041085:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878914.041093:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878914.041094:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041097:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878914.041099:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878914.041102:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041112:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878914.041119:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290110848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878914.041132:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878914.041135:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041145:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878914.041148:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878914.041150:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878914.041151:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878914.041152:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878914.041154:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878914.041156:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9852 00000100:00080000:0.0:1788878914.041159:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878914.041160:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878914.041161:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878914.041164:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878914.041166:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878914.041167:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788878914.041169:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878914.041170:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878914.041171:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878914.041173:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878914.041175:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041175:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788878914.041179:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788878914.041181:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788878914.041184:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878914.041185:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1788878914.041188:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041190:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878914.041191:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878914.041195:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788878914.041196:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878914.041201:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041204:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878914.041204:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878914.041209:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878914.041210:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878914.041212:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041216:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878914.041217:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878914.041217:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878914.041220:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041225:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878914.041227:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290110848:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1788878914.041232:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290110976/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878914.041235:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788878914.041242:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788878914.041242:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788878914.041246:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878914.041247:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041249:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878914.041252:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1788878914.041252:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1788878914.041257:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878914.041258:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878914.041259:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1788878914.041260:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041261:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878914.041265:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9852 00000100:00080000:0.0:1788878914.041267:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878914.041268:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878914.041270:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878914.041272:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878914.041272:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878914.041274:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878914.041275:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878914.041276:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878914.041277:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878914.041279:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878914.041281:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878914.041282:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878914.041284:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878914.041285:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041286:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878914.041287:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878914.041289:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041291:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878914.041294:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878914.041296:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1788878914.041297:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041299:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:0.0:1788878914.041301:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878914.041304:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1788878914.041305:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878914.041309:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041316:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878914.041322:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290111104/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878914.041330:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878914.041333:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041335:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788878914.041340:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1788878914.041341:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878914.041347:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878914.041349:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878914.041351:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878914.041353:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878914.041354:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788878914.041359:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1788878914.041360:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9852 00000100:00000001:2.0:1788878914.041363:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041366:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:0.0:1788878914.041367:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878914.041370:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878914.041370:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041373:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878914.041376:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304525865 00000020:00000001:0.0:1788878914.041377:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878914.041380:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878914.041382:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788878914.041384:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878914.041386:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878914.041387:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878914.041387:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1788878914.041391:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878914.041394:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041395:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304525865 02000000:00000001:0.0:1788878914.041396:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878914.041401:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878914.041401:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1788878914.041407:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041408:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878914.041409:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878914.041411:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041413:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041413:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878914.041417:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878914.041417:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041419:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1788878914.041420:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041423:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878914.041424:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1788878914.041427:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878914/real 1788878914] req@ffff94e50897ad80 x1875765290110848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788878914.041428:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.041436:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788878914.041442:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:0.0:1788878914.041445:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290111232/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1788878914.041446:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878914.041450:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878914.041454:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878914.041455:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878914.041458:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878914.041459:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041462:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878914.041463:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041464:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878914.041471:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290110848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878914.041476:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041488:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878914.041491:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041493:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878914.041494:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041496:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878914.041497:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878914.041499:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878914.041503:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878914.041504:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878914.041506:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290110848:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878914.041510:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041512:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878914.041513:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878914.041515:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878914.041517:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878914.041536:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878914.041538:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878914.041542:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878914.041543:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878914.041547:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878914.041549:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041551:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041558:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878914.041560:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041562:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878914.041563:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878914.041565:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878914.041567:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878914.041568:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878914.041570:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878914.041572:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878914.041575:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878914.041578:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290110976:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878914.041581:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878914.041583:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878914.041585:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878914.041586:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878914.041589:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041596:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878914.041597:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:2.0:1788878914.041602:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878914.041616:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878914.041620:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878914.041623:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878914.041624:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041626:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878914.041628:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041632:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304525323 00000100:00000001:2.0:1788878914.041635:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304525865 00000100:00000001:2.0:1788878914.041637:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304525323 00000100:00000001:2.0:1788878914.041640:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304525865 00000100:00000001:2.0:1788878914.041642:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878914.041644:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041646:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878914.041650:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878914/real 1788878914] req@ffff94e50897bb80 x1875765290110976/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878914.041659:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878914.041660:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878914.041662:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878914.041664:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878914.041665:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041667:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878914.041668:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878914.041672:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290110976/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878914.041679:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041681:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878914.041683:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041685:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878914.041686:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041687:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878914.041699:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878914.041701:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878914.041704:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878914.041706:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878914.041708:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290110976:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878914.041712:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878914.041713:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878914.041714:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878914.041716:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878914.041718:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878914.041720:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878914.041723:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290111104:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878914.041726:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878914.041727:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878914.041729:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878914.041730:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878914.041733:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041737:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878914.041738:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:2.0:1788878914.041741:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878914.041749:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878914.041753:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878914.041754:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878914.041756:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041757:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878914.041760:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041762:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304525321 00000100:00000001:2.0:1788878914.041765:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304525865 00000100:00000001:2.0:1788878914.041767:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304525321 00000100:00000001:2.0:1788878914.041769:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304525865 00000100:00000001:2.0:1788878914.041771:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878914.041773:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041774:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878914.041778:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878914/real 1788878914] req@ffff94e508978e00 x1875765290111104/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878914.041785:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878914.041786:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878914.041788:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878914.041789:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878914.041790:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041791:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878914.041793:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878914.041796:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290111104/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878914.041803:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041816:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878914.041818:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041820:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878914.041821:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041822:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878914.041832:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878914.041834:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878914.041836:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878914.041838:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878914.041842:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290111104:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878914.041847:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878914.041849:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878914.041851:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878914.041854:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878914.041856:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878914.041860:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878914.041865:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290111232:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878914.041885:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878914.041886:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878914.041888:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878914.041890:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878914.041893:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041898:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878914.041900:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:2.0:1788878914.041903:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878914.041914:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878914.041918:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878914.041921:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878914.041923:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041924:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878914.041927:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041930:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304525321 00000100:00000001:2.0:1788878914.041932:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304525865 00000100:00000001:2.0:1788878914.041949:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304525322 00000100:00000001:2.0:1788878914.041951:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304525865 00000100:00000001:2.0:1788878914.041952:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878914.041953:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.041955:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878914.041958:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878914/real 1788878914] req@ffff94e508978a80 x1875765290111232/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878914.041964:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878914.041965:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878914.041967:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878914.041968:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878914.041969:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041970:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878914.041971:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878914.041973:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290111232/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878914.041978:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041979:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878914.041980:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041981:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878914.041982:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.041983:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878914.041991:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878914.041993:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878914.041995:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878914.041997:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878914.041999:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290111232:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878914.042001:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.042003:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878914.042004:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878914.042005:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878914.042006:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878914.042007:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878914.042008:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878914.042010:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878914.042011:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878914.042012:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878914.042013:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.042014:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878914.042015:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878914.042016:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878914.042017:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878914.042018:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878914.042019:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878914.042020:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878914.042021:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878914.042022:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878914.042023:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.042024:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878914.042025:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878914.042025:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878914.042026:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878914.042027:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878914.042027:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878914.042029:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878914.042029:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878914.042030:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878914.042031:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.042032:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878914.042034:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878914.042035:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878914.042036:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878914.042037:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878914.105003:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878914.105010:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878914.105012:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878914.105016:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878914.105019:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878914.105022:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878914.105025:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878914.105027:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878914.105029:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878914.809027:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878914.809037:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878914.809040:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878914.809044:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878914.809049:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878914.809052:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878914.809054:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878914.809056:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878914.809060:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878914.809063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878914.809098:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878914.809102:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878914.809106:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878915.128979:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878915.128985:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878915.128986:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878915.128990:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878915.128993:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878915.128996:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878915.128999:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878915.129000:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878915.129002:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878915.832991:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878915.832997:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878915.833003:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878915.833004:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878915.833030:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878915.833033:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878915.833036:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878916.152970:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878916.152976:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878916.152977:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878916.152982:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878916.152984:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878916.152988:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878916.152990:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878916.152992:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878916.152994:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878916.856987:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878916.856992:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878916.856996:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878916.856997:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878916.857054:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878916.857058:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878916.857061:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878917.176974:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878917.176980:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878917.176981:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878917.176985:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878917.176988:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878917.176991:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878917.176993:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878917.176996:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878917.176998:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878917.881007:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878917.881013:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878917.881019:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878917.881021:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878917.881132:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878917.881137:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878917.881140:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878918.201006:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878918.201013:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878918.201015:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878918.201020:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878918.201024:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878918.201028:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878918.201031:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878918.201033:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878918.201036:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878918.904995:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878918.905000:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878918.905005:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878918.905006:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878918.905044:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878918.905047:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878918.905049:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878919.161020:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878919.161032:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878919.161035:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878919.161039:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878919.161044:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878919.161052:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9858 00000100:00080000:0.0:1788878919.161064:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878919.161067:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878919.161070:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878919.161078:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878919.161081:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878919.161086:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878919.161089:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878919.161091:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878919.161093:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878919.161098:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878919.161100:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878919.161104:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878919.161107:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878919.161110:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878919.161113:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878919.161115:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878919.161128:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878919.161130:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878919.161133:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878919.161136:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878919.161141:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878919.161154:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878919.161164:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978700 x1875765290111360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878919.161175:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878919.161181:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878919.161199:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878919.161203:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878919.161207:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878919.161209:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878919.161211:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878919.161214:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878919.161217:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9858 00000100:00080000:0.0:1788878919.161220:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878919.161222:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878919.161223:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878919.161226:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878919.161228:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878919.161231:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788878919.161233:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878919.161235:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878919.161237:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878919.161240:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878919.161242:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878919.161244:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878919.161250:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878919.161252:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878919.161254:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878919.161258:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878919.161261:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878919.161264:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878919.161268:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878919.161270:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878919.161272:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878919.161274:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878919.161277:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878919.161283:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878919.161290:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897aa00 x1875765290111488/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878919.161296:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878919.161302:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878919.161305:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878919.161308:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.161311:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878919.161315:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.161317:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878919.161323:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878919.161326:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1788878919.161330:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878919.161336:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1788878919.161339:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:2.0:1788878919.161341:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878919.161347:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1788878919.161354:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290111360:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1788878919.161356:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878919.161365:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788878919.161369:0:80852:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:2.0:1788878919.161372:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878919.161377:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878919.161382:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9858 02000000:00000001:2.0:1788878919.161383:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1788878919.161394:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878919.161399:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878919.161402:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788878919.161408:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878919.161409:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878919.161414:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878919.161418:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878919.161421:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878919.161424:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878919.161426:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788878919.161432:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1788878919.161433:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878919.161438:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00000001:0.0:1788878919.161440:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.161470:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1788878919.161524:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878919.161530:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878919.161535:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788878919.161539:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878919.161540:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878919.161546:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878919.161551:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878919.161553:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.161560:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:0.0:1788878919.161561:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878919.161566:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.161569:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878919.161570:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878919.161575:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878919.161576:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878919.161580:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.161582:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878919.161592:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878919.161593:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.161599:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304530985 00000100:00100000:0.0:1788878919.161606:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290111616/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878919.161618:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878919.161618:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878919.161625:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878919.161630:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878919.161633:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788878919.161636:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304530985 00000100:00080000:0.0:1788878919.161637:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878919.161642:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878919.161647:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878919.161650:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878919.161652:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878919.161660:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9858 00000100:00080000:0.0:1788878919.161666:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878919.161668:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878919.161670:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878919.161674:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788878919.161676:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788878919.161677:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878919.161682:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878919.161688:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878919.161689:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878919.161696:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788878919.161706:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878919.161711:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:2.0:1788878919.161713:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878919/real 1788878919] req@ffff94e508978700 x1875765290111360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878919.161721:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878919.161735:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878919.161744:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:2.0:1788878919.161745:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878919.161749:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1788878919.161751:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878919.161756:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:0.0:1788878919.161759:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.161762:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878919.161766:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878919.161767:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1788878919.161776:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788878919.161776:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878919.161781:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788878919.161789:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878919.161794:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290111360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878919.161797:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.161813:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788878919.161814:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878919.161818:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1788878919.161819:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878919.161828:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788878919.161830:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788878919.161832:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788878919.161835:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1788878919.161837:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.161839:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.161845:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878919.161848:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.161849:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878919.161859:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1788878919.161866:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290111744/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788878919.161870:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878919.161877:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878919.161889:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290111360:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878919.161892:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878919.161898:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.161899:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.161941:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878919.161943:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.161944:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878919.161951:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878919.161959:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878919.161962:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878919.161964:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878919.161970:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878919.161973:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878919.161976:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878919.161978:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.161982:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.161989:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878919.161991:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.161994:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878919.161996:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878919.161998:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878919.162000:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878919.162001:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878919.162004:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878919.162007:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878919.162010:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878919.162014:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290111488:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878919.162018:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878919.162020:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878919.162022:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878919.162024:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878919.162028:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.162035:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878919.162037:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:2.0:1788878919.162042:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878919.162058:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878919.162064:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878919.162067:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878919.162068:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.162070:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878919.162073:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.162078:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304530444 00000100:00000001:2.0:1788878919.162081:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304530985 00000100:00000001:2.0:1788878919.162084:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304530444 00000100:00000001:2.0:1788878919.162086:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304530985 00000100:00000001:2.0:1788878919.162089:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878919.162091:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.162093:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878919.162097:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878919/real 1788878919] req@ffff94e50897aa00 x1875765290111488/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878919.162106:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878919.162107:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878919.162110:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878919.162112:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878919.162113:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162115:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878919.162117:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878919.162121:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290111488/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878919.162129:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162131:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878919.162134:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162136:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878919.162137:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162139:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878919.162151:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878919.162154:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878919.162157:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878919.162160:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878919.162163:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290111488:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878919.162167:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878919.162168:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878919.162170:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878919.162173:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878919.162174:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878919.162177:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878919.162179:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290111616:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878919.162182:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878919.162184:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878919.162185:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878919.162187:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878919.162190:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.162194:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878919.162196:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:2.0:1788878919.162200:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878919.162209:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878919.162213:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878919.162216:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878919.162218:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.162219:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878919.162222:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.162225:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304530442 00000100:00000001:2.0:1788878919.162228:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304530985 00000100:00000001:2.0:1788878919.162230:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304530442 00000100:00000001:2.0:1788878919.162232:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304530985 00000100:00000001:2.0:1788878919.162235:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878919.162237:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.162238:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878919.162242:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878919/real 1788878919] req@ffff94e50897a300 x1875765290111616/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878919.162250:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878919.162251:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878919.162253:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878919.162255:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878919.162256:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162258:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878919.162259:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878919.162263:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290111616/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878919.162270:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162273:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878919.162275:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162276:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878919.162278:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162279:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878919.162289:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878919.162292:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878919.162294:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878919.162296:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878919.162300:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290111616:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878919.162304:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878919.162306:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878919.162307:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878919.162309:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878919.162311:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878919.162313:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878919.162316:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290111744:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878919.162319:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878919.162320:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878919.162322:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878919.162323:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878919.162326:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.162330:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878919.162331:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:2.0:1788878919.162335:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878919.162344:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878919.162347:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878919.162349:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878919.162351:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.162352:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878919.162355:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.162358:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304530442 00000100:00000001:2.0:1788878919.162361:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304530985 00000100:00000001:2.0:1788878919.162363:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304530442 00000100:00000001:2.0:1788878919.162366:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304530985 00000100:00000001:2.0:1788878919.162368:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878919.162370:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.162372:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878919.162376:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878919/real 1788878919] req@ffff94e508978a80 x1875765290111744/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878919.162390:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878919.162392:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878919.162394:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878919.162396:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878919.162397:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162399:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878919.162400:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878919.162404:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290111744/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878919.162410:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162412:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878919.162415:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162416:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878919.162417:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162419:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878919.162430:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878919.162432:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878919.162435:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878919.162437:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878919.162440:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290111744:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878919.162464:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162467:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878919.162468:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878919.162470:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878919.162473:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878919.162474:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878919.162476:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878919.162479:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878919.162480:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878919.162483:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878919.162485:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162487:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878919.162489:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878919.162490:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878919.162492:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878919.162494:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878919.162495:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878919.162497:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878919.162498:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878919.162499:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878919.162501:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162502:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878919.162503:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878919.162504:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878919.162506:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878919.162507:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878919.162508:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878919.162510:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878919.162511:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878919.162512:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878919.162514:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162516:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878919.162519:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878919.162521:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878919.162523:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878919.162524:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878919.224990:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878919.224997:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878919.225000:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878919.225006:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878919.225010:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878919.225013:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878919.225016:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878919.225019:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878919.225022:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878919.928968:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878919.928973:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878919.928976:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878919.928978:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878919.928993:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878919.928995:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878919.928997:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878920.249003:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878920.249010:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878920.249012:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878920.249018:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878920.249022:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878920.249026:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878920.249029:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878920.249032:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878920.249034:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878920.953025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878920.953033:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878920.953040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878920.953042:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878920.953133:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878920.953143:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878920.953147:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878921.273048:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878921.273057:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878921.273060:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878921.273066:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878921.273071:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878921.273075:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878921.273078:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878921.273081:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878921.273085:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878921.977027:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878921.977035:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878921.977041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878921.977043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878921.977185:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878921.977191:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878921.977194:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878922.296968:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878922.296974:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878922.296975:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878922.296980:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878922.296983:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878922.296985:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878922.296987:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878922.296989:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878922.296991:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878923.000967:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878923.000972:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878923.000976:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878923.000979:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878923.000981:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878923.000983:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878923.000985:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878923.000987:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878923.000989:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878923.000990:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878923.001021:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878923.001023:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878923.001025:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878923.320983:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878923.320987:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878923.320988:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878923.320992:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878923.320994:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878923.320997:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878923.320999:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878923.321001:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878923.321003:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878924.024969:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878924.024974:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878924.024977:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878924.024979:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878924.024995:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878924.024997:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878924.024999:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878924.281009:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878924.281017:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878924.281019:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878924.281022:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878924.281025:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878924.281030:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9863 00000100:00080000:0.0:1788878924.281047:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878924.281049:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878924.281051:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878924.281058:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878924.281060:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878924.281063:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878924.281065:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878924.281066:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878924.281067:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878924.281070:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878924.281072:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878924.281074:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878924.281077:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878924.281079:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878924.281080:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878924.281082:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878924.281092:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878924.281093:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878924.281095:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878924.281098:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878924.281101:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878924.281111:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878924.281118:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290111872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878924.281131:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878924.281134:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878924.281145:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878924.281148:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878924.281150:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878924.281151:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878924.281153:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878924.281154:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878924.281156:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9863 00000100:00080000:0.0:1788878924.281158:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878924.281159:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878924.281160:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878924.281162:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878924.281163:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878924.281164:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788878924.281166:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878924.281167:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878924.281168:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878924.281170:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878924.281171:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878924.281172:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878924.281176:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878924.281177:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878924.281178:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878924.281181:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878924.281183:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878924.281185:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878924.281187:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878924.281188:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878924.281190:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878924.281191:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878924.281192:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878924.281195:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878924.281199:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290112000/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878924.281205:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878924.281206:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878924.281208:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281209:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1788878924.281210:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788878924.281214:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788878924.281215:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878924.281218:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788878924.281219:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1788878924.281220:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878924.281222:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878924.281223:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788878924.281225:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878924.281228:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9863 02000000:00000001:2.0:1788878924.281231:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1788878924.281232:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878924.281235:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1788878924.281235:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878924.281236:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788878924.281240:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878924.281241:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878924.281244:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878924.281245:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878924.281248:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00100000:2.0:1788878924.281249:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290111872:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878924.281250:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878924.281253:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878924.281259:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878924.281260:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788878924.281262:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878924.281264:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878924.281266:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878924.281267:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878924.281270:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878924.281271:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878924.281286:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788878924.281287:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878924.281290:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788878924.281292:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788878924.281293:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878924.281297:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878924.281300:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878924.281301:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878924.281302:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878924.281304:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878924.281305:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878924.281309:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281310:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00100000:0.0:1788878924.281313:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290112128/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878924.281315:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:2.0:1788878924.281322:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788878924.281323:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878924.281326:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878924.281328:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878924.281330:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878924.281332:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878924.281333:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878924.281334:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878924.281336:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878924.281338:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9863 00000100:00080000:0.0:1788878924.281340:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878924.281341:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878924.281342:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878924.281350:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878924.281351:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878924.281352:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788878924.281354:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878924.281355:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878924.281357:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:2.0:1788878924.281359:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1788878924.281362:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878924.281365:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878924.281367:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878924.281370:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878924.281371:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878924.281373:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878924.281373:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00100000:0.0:1788878924.281377:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878924.281390:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281391:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878924.281393:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281395:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281397:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:0.0:1788878924.281403:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788878924.281403:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878924.281406:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878924.281421:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878924.281422:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878924.281423:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281424:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304536105 00000100:00000001:0.0:1788878924.281427:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878924.281431:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290112256/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878924.281439:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878924.281441:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281443:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878924.281444:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281453:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304536105 00000100:00000001:2.0:1788878924.281457:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788878924.281466:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878924.281468:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281470:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878924.281475:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878924/real 1788878924] req@ffff94e508978e00 x1875765290111872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878924.281484:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878924.281486:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878924.281491:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878924.281493:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878924.281495:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281497:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878924.281499:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878924.281502:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290111872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878924.281507:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281510:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878924.281511:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281512:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878924.281513:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281515:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878924.281516:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878924.281518:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878924.281521:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878924.281523:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878924.281526:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290111872:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878924.281529:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878924.281530:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878924.281531:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878924.281532:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878924.281533:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878924.281535:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878924.281537:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290112000:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878924.281539:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878924.281541:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878924.281542:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878924.281543:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878924.281545:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281549:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878924.281550:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:2.0:1788878924.281553:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878924.281562:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878924.281566:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878924.281567:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878924.281568:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281569:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878924.281571:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281574:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304535563 00000100:00000001:2.0:1788878924.281576:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304536105 00000100:00000001:2.0:1788878924.281578:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304535563 00000100:00000001:2.0:1788878924.281579:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304536105 00000100:00000001:2.0:1788878924.281581:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878924.281582:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281583:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878924.281586:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878924/real 1788878924] req@ffff94e50897bb80 x1875765290112000/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878924.281592:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878924.281592:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878924.281594:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878924.281595:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878924.281596:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281597:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878924.281598:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878924.281600:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290112000/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878924.281604:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281605:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878924.281607:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281608:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878924.281609:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281610:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878924.281619:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878924.281621:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878924.281623:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878924.281624:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878924.281626:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290112000:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878924.281628:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281630:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878924.281631:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878924.281633:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878924.281635:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878924.281637:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878924.281639:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878924.281641:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878924.281642:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878924.281646:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878924.281646:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281648:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878924.281648:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878924.281649:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878924.281651:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878924.281652:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878924.281652:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878924.281654:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878924.281654:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878924.281656:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878924.281657:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281658:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281663:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878924.281664:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281666:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878924.281667:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878924.281667:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878924.281668:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878924.281669:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878924.281671:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878924.281672:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878924.281673:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878924.281675:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290112128:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878924.281677:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878924.281678:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878924.281679:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878924.281680:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878924.281682:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281685:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878924.281686:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:2.0:1788878924.281689:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878924.281695:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878924.281697:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878924.281698:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878924.281699:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281701:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878924.281702:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281711:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304535561 00000100:00000001:2.0:1788878924.281713:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304536105 00000100:00000001:2.0:1788878924.281715:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304535561 00000100:00000001:2.0:1788878924.281716:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304536105 00000100:00000001:2.0:1788878924.281718:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878924.281719:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281720:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878924.281722:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878924/real 1788878924] req@ffff94e50897ad80 x1875765290112128/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878924.281727:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878924.281728:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878924.281730:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878924.281731:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878924.281732:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281733:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878924.281734:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878924.281736:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290112128/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878924.281741:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281742:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878924.281744:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281745:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878924.281745:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281746:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878924.281754:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878924.281756:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878924.281757:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878924.281759:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878924.281760:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290112128:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878924.281762:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878924.281763:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878924.281764:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878924.281766:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878924.281767:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878924.281768:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878924.281770:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290112256:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878924.281772:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788878924.281773:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788878924.281774:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878924.281775:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788878924.281777:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281779:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878924.281780:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:2.0:1788878924.281783:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788878924.281789:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878924.281791:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878924.281793:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878924.281793:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281794:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878924.281796:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281798:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304535561 00000100:00000001:2.0:1788878924.281800:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304536105 00000100:00000001:2.0:1788878924.281801:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304535561 00000100:00000001:2.0:1788878924.281803:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304536105 00000100:00000001:2.0:1788878924.281804:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878924.281805:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281806:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788878924.281808:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878924/real 1788878924] req@ffff94e508979c00 x1875765290112256/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788878924.281813:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788878924.281814:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878924.281816:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878924.281817:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878924.281818:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281819:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878924.281820:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788878924.281822:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290112256/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878924.281826:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281828:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878924.281829:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281830:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878924.281831:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281832:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878924.281839:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788878924.281841:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788878924.281842:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788878924.281844:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878924.281845:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290112256:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788878924.281847:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281848:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878924.281849:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878924.281850:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878924.281852:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878924.281853:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878924.281853:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878924.281855:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878924.281856:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878924.281858:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878924.281859:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281860:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878924.281861:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788878924.281862:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788878924.281862:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788878924.281863:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788878924.281864:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788878924.281865:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788878924.281866:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878924.281867:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878924.281868:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281869:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878924.281871:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878924.281872:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878924.281873:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788878924.281874:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878924.344957:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878924.344961:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878924.344962:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878924.344966:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878924.344968:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878924.344970:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878924.344972:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878924.344974:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878924.344976:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878925.048966:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878925.048973:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878925.048975:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878925.048977:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878925.048981:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878925.048983:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878925.048984:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878925.048986:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878925.048988:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878925.048989:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878925.049051:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878925.049055:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878925.049059:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878925.369006:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878925.369011:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878925.369012:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878925.369016:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878925.369019:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878925.369022:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878925.369024:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878925.369026:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878925.369028:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878926.073015:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878926.073020:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878926.073023:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878926.073024:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878926.073098:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878926.073101:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878926.073104:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878926.392962:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878926.392968:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878926.392970:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878926.392975:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878926.392979:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878926.392982:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878926.392993:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878926.392996:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878926.393000:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878927.096978:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878927.096983:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878927.096987:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878927.096988:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878927.097006:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878927.097007:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878927.097010:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878927.417012:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878927.417019:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878927.417021:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878927.417027:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878927.417030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878927.417035:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878927.417038:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878927.417040:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878927.417044:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878928.121083:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878928.121093:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878928.121102:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878928.121106:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878928.121157:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878928.121162:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878928.121168:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878928.441000:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878928.441006:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878928.441008:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878928.441013:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878928.441017:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878928.441020:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878928.441023:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878928.441025:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878928.441027:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878929.145042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878929.145049:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878929.145055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878929.145057:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878929.145133:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878929.145139:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878929.145142:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878929.400993:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878929.401004:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878929.401007:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878929.401010:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878929.401015:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878929.401023:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9868 00000100:00080000:2.0:1788878929.401035:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878929.401038:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878929.401040:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878929.401048:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878929.401051:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878929.401054:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878929.401057:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878929.401059:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878929.401060:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878929.401065:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878929.401068:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878929.401072:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878929.401075:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878929.401078:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878929.401080:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878929.401083:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878929.401093:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878929.401095:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878929.401098:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878929.401101:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878929.401105:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878929.401118:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878929.401128:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290112384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878929.401139:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878929.401144:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878929.401163:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878929.401167:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878929.401170:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878929.401172:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878929.401174:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878929.401178:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878929.401181:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9868 00000100:00000001:1.0:1788878929.401187:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788878929.401187:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878929.401192:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878929.401192:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878929.401196:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788878929.401197:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878929.401199:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878929.401202:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878929.401203:0:178134:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1788878929.401206:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1788878929.401207:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1788878929.401210:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1788878929.401212:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:1.0:1788878929.401214:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878929.401219:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00100000:1.0:1788878929.401222:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290112384:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878929.401224:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878929.401228:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878929.401229:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878929.401233:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878929.401236:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878929.401238:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788878929.401240:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878929.401246:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878929.401255:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878929.401259:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878929.401261:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878929.401263:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 02000000:00000001:2.0:1788878929.401266:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878929.401267:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1788878929.401272:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878929.401275:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878929.401279:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878929.401282:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878929.401284:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878929.401290:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878929.401292:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878929.401294:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878929.401295:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878929.401297:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788878929.401301:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878929.401307:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878929.401316:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290112512/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878929.401325:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878929.401328:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878929.401332:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878929.401335:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788878929.401337:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788878929.401339:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788878929.401343:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878929.401343:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878929.401346:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878929.401347:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401350:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788878929.401354:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788878929.401356:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878929.401360:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9868 00000100:00000001:1.0:1788878929.401361:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304541225 00000100:00080000:2.0:1788878929.401369:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878929.401387:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878929.401388:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878929.401396:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878929.401397:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304541225 00000020:00000001:2.0:1788878929.401405:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788878929.401406:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1788878929.401409:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788878929.401412:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1788878929.401414:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788878929.401417:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878929.401419:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788878929.401422:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878929.401424:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878929.401428:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:1.0:1788878929.401432:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878929/real 1788878929] req@ffff94e5045e6300 x1875765290112384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878929.401436:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878929.401442:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878929.401447:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878929.401452:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:1.0:1788878929.401456:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1788878929.401457:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788878929.401458:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788878929.401462:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788878929.401464:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000008:00100000:2.0:1788878929.401467:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878929.401468:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878929.401470:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1788878929.401473:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401474:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878929.401476:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878929.401479:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878929.401482:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290112384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788878929.401486:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878929.401490:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401491:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878929.401494:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788878929.401496:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788878929.401498:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788878929.401500:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788878929.401501:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401502:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878929.401505:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401508:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878929.401509:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878929.401511:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788878929.401512:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878929.401519:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878929.401522:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788878929.401524:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290112640/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788878929.401527:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290112384:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878929.401536:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401540:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878929.401544:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788878929.401544:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788878929.401549:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788878929.401550:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878929.401552:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878929.401555:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:2.0:1788878929.401555:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878929.401558:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00080000:2.0:1788878929.401561:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788878929.401563:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878929.401566:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:2.0:1788878929.401568:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788878929.401569:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878929.401571:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401572:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401577:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788878929.401579:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788878929.401580:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401584:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788878929.401584:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878929.401587:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878929.401589:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878929.401592:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878929.401592:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788878929.401596:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878929.401598:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1788878929.401599:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9868 02000000:00000001:1.0:1788878929.401600:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878929.401605:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878929.401608:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788878929.401609:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290112512:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788878929.401613:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878929.401615:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788878929.401617:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788878929.401618:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878929.401620:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878929.401623:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1788878929.401624:0:178134:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1788878929.401626:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788878929.401628:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788878929.401632:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1788878929.401633:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788878929.401635:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000020:00000001:2.0:1788878929.401639:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788878929.401641:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788878929.401645:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878929.401647:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878929.401652:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:1.0:1788878929.401656:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878929.401660:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401662:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:2.0:1788878929.401665:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878929.401666:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878929.401669:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401671:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:2.0:1788878929.401672:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878929.401674:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878929.401678:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401679:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304540683 00000100:00000001:2.0:1788878929.401682:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788878929.401684:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304541225 00000008:00100000:2.0:1788878929.401685:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878929.401687:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304540683 00000008:00000001:2.0:1788878929.401691:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401692:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304541225 00000100:00000001:1.0:1788878929.401696:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878929.401698:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401699:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401702:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1788878929.401705:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1788878929.401706:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878929/real 1788878929] req@ffff94e5045e7800 x1875765290112512/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788878929.401709:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878929.401715:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878929.401719:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:1.0:1788878929.401720:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1788878929.401722:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788878929.401723:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878929.401726:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878929.401728:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878929.401729:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401731:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788878929.401731:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401733:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878929.401739:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290112512/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788878929.401743:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290112768/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878929.401748:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401752:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878929.401757:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401760:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878929.401762:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401765:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788878929.401767:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878929.401770:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878929.401776:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401781:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878929.401785:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878929.401788:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878929.401790:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878929.401793:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290112512:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878929.401797:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878929.401798:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878929.401799:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878929.401801:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878929.401802:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878929.401804:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878929.401806:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290112640:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878929.401808:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878929.401809:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878929.401809:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878929.401810:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878929.401812:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401815:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878929.401816:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:1.0:1788878929.401818:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878929.401824:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878929.401827:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878929.401828:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878929.401829:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401830:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878929.401832:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401834:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304540681 00000100:00000001:1.0:1788878929.401836:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304541225 00000100:00000001:1.0:1788878929.401837:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304540681 00000100:00000001:1.0:1788878929.401839:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304541225 00000100:00000001:1.0:1788878929.401840:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878929.401841:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401843:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878929.401845:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878929/real 1788878929] req@ffff94e5045e7100 x1875765290112640/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878929.401850:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878929.401851:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878929.401852:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878929.401853:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878929.401854:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401855:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878929.401856:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878929.401858:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290112640/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878929.401863:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401864:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878929.401865:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401866:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878929.401867:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401868:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878929.401876:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878929.401878:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878929.401879:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878929.401881:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878929.401882:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290112640:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878929.401884:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401886:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878929.401886:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878929.401887:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878929.401889:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878929.401890:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878929.401891:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878929.401893:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878929.401893:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878929.401895:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878929.401896:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401897:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878929.401897:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878929.401898:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878929.401899:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878929.401900:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878929.401901:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878929.401902:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878929.401923:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878929.401925:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878929.401925:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401927:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401929:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878929.401930:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401931:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878929.401932:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878929.401933:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878929.401934:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878929.401935:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878929.401936:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878929.401937:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878929.401939:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878929.401941:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290112768:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878929.401943:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878929.401944:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878929.401945:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878929.401946:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878929.401948:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401951:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878929.401952:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:1.0:1788878929.401955:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878929.401962:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878929.401964:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878929.401966:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878929.401967:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401968:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878929.401970:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401972:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304540682 00000100:00000001:1.0:1788878929.401973:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304541225 00000100:00000001:1.0:1788878929.401975:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304540682 00000100:00000001:1.0:1788878929.401976:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304541225 00000100:00000001:1.0:1788878929.401978:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878929.401979:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.401980:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878929.401982:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878929/real 1788878929] req@ffff94e5045e4a80 x1875765290112768/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878929.401987:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878929.401988:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878929.401989:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878929.401990:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878929.401991:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.401992:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878929.401993:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878929.401996:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290112768/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878929.402000:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.402002:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878929.402003:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.402004:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878929.402005:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.402006:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878929.402014:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878929.402016:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878929.402017:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878929.402018:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878929.402021:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290112768:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878929.402023:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.402024:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878929.402025:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878929.402026:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878929.402027:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878929.402028:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878929.402029:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878929.402030:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878929.402031:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878929.402032:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878929.402033:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.402034:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878929.402035:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878929.402036:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878929.402038:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878929.402039:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878929.465018:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878929.465024:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878929.465025:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878929.465030:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878929.465033:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878929.465036:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878929.465039:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878929.465041:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878929.465043:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878930.169054:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878930.169059:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878930.169062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878930.169063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878930.169151:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878930.169155:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878930.169158:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878930.488976:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878930.488982:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878930.488984:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878930.488989:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878930.488993:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878930.488997:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878930.489000:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878930.489003:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878930.489006:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878931.193034:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878931.193039:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878931.193043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878931.193044:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878931.193097:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878931.193099:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878931.193102:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878931.512993:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878931.512999:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878931.513001:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878931.513005:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878931.513008:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878931.513011:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878931.513013:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878931.513016:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878931.513018:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878932.217032:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878932.217038:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878932.217044:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878932.217046:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878932.217049:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878932.217052:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878932.217055:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878932.217057:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878932.217059:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878932.217062:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878932.217083:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878932.217086:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878932.217088:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878932.537027:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878932.537036:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878932.537039:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878932.537046:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878932.537052:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878932.537057:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878932.537061:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878932.537065:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878932.537068:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878933.241032:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878933.241039:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878933.241045:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878933.241046:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878933.241172:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878933.241178:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878933.241182:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878933.561058:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878933.561066:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878933.561069:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878933.561075:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878933.561080:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878933.561084:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878933.561087:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878933.561090:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878933.561094:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878934.264976:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878934.264981:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878934.264985:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878934.264987:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878934.265101:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878934.265105:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878934.265108:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878934.520997:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878934.521008:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878934.521011:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878934.521014:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878934.521019:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878934.521026:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9873 00000100:00080000:2.0:1788878934.521036:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878934.521039:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878934.521041:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878934.521049:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878934.521052:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878934.521055:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878934.521058:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878934.521060:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878934.521062:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878934.521067:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878934.521070:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878934.521074:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878934.521078:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878934.521083:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521086:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878934.521104:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878934.521114:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878934.521117:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521120:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878934.521123:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878934.521127:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521139:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878934.521149:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290112896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878934.521173:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878934.521177:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521195:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878934.521199:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878934.521202:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878934.521204:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878934.521205:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878934.521208:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878934.521210:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9873 00000100:00080000:2.0:1788878934.521214:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878934.521217:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878934.521218:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878934.521221:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878934.521223:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878934.521225:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878934.521227:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878934.521229:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878934.521231:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878934.521234:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878934.521236:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878934.521237:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878934.521242:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878934.521244:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521246:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878934.521250:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878934.521253:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521255:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878934.521259:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878934.521261:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521263:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878934.521266:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878934.521269:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521288:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.521289:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878934.521296:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1788878934.521298:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290113024/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878934.521301:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878934.521307:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878934.521313:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521321:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788878934.521351:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788878934.521354:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878934.521357:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788878934.521360:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878934.521373:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878934.521376:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878934.521380:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878934.521382:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788878934.521383:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290112896:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1788878934.521384:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878934.521405:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788878934.521411:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1788878934.521412:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9873 02000000:00000001:1.0:1788878934.521420:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1788878934.521420:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878934.521426:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788878934.521427:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521428:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878934.521435:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878934.521436:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878934.521439:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878934.521441:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878934.521444:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788878934.521446:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788878934.521447:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878934.521453:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788878934.521455:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521456:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878934.521464:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878934.521467:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878934.521469:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521472:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878934.521474:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878934.521477:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.521478:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788878934.521479:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.521483:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 02000000:00000001:2.0:1788878934.521488:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788878934.521490:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1788878934.521492:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521495:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878934.521497:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878934.521500:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521509:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878934.521517:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290113152/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878934.521529:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000100:1.0:1788878934.521552:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878934.521552:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521557:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878934.521562:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878934.521566:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788878934.521567:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788878934.521568:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788878934.521572:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788878934.521573:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878934.521575:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521577:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788878934.521578:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:2.0:1788878934.521582:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9873 00000100:00000001:1.0:1788878934.521585:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878934.521587:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878934.521589:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304546345 00000100:00000001:2.0:1788878934.521590:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878934.521594:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878934.521597:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878934.521598:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788878934.521600:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1788878934.521600:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788878934.521603:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788878934.521605:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878934.521607:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878934.521625:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304546345 00000100:00080000:2.0:1788878934.521627:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878934.521634:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.521635:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1788878934.521637:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878934.521641:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878934.521642:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878934.521645:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878934.521647:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.521651:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878934.521652:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878934.521656:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878934.521662:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878934.521663:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878934/real 1788878934] req@ffff94e5045e5180 x1875765290112896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878934.521665:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878934.521673:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878934.521675:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521681:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:1.0:1788878934.521684:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1788878934.521684:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878934.521688:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788878934.521689:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878934.521696:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788878934.521699:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.521700:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878934.521705:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.521719:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:2.0:1788878934.521720:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290113280/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878934.521724:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878934.521737:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290112896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878934.521737:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878934.521743:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878934.521746:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.521760:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.521764:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878934.521767:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.521769:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878934.521771:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.521774:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878934.521776:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878934.521779:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878934.521794:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878934.521796:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878934.521800:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290112896:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878934.521804:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.521806:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878934.521807:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878934.521826:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878934.521828:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878934.521830:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878934.521832:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878934.521835:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878934.521836:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878934.521838:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878934.521839:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.521841:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.521847:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878934.521848:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.521850:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878934.521851:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878934.521852:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878934.521853:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878934.521854:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878934.521855:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878934.521857:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878934.521858:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878934.521860:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290113024:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878934.521862:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878934.521863:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878934.521864:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878934.521866:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878934.521868:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.521872:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878934.521873:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:1.0:1788878934.521894:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878934.521907:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878934.521911:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878934.521913:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878934.521915:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.521916:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878934.521919:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.521922:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304545803 00000100:00000001:1.0:1788878934.521925:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304546345 00000100:00000001:1.0:1788878934.521927:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304545803 00000100:00000001:1.0:1788878934.521930:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304546345 00000100:00000001:1.0:1788878934.521932:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878934.521964:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.521966:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878934.521970:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878934/real 1788878934] req@ffff94e5045e5880 x1875765290113024/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878934.521978:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878934.521989:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878934.521992:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878934.521993:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878934.521995:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.521996:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878934.521998:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878934.522001:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290113024/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878934.522007:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.522009:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878934.522011:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.522013:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878934.522014:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.522015:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878934.522027:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878934.522030:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878934.522033:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878934.522035:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878934.522038:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290113024:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878934.522041:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878934.522042:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878934.522044:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878934.522045:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878934.522047:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878934.522049:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878934.522051:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290113152:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878934.522055:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878934.522056:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878934.522058:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878934.522059:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878934.522062:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.522067:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878934.522068:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:1.0:1788878934.522071:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878934.522081:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878934.522085:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878934.522087:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878934.522089:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.522090:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878934.522093:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.522122:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304545802 00000100:00000001:1.0:1788878934.522123:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304546345 00000100:00000001:1.0:1788878934.522125:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304545802 00000100:00000001:1.0:1788878934.522127:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304546345 00000100:00000001:1.0:1788878934.522128:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878934.522129:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.522131:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878934.522133:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878934/real 1788878934] req@ffff94e5045e6a00 x1875765290113152/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878934.522139:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878934.522139:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878934.522141:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878934.522142:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878934.522143:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.522144:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878934.522145:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878934.522147:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290113152/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878934.522151:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.522153:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878934.522154:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.522155:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878934.522156:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.522157:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878934.522165:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878934.522166:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878934.522168:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878934.522186:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878934.522189:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290113152:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878934.522202:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878934.522203:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878934.522205:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878934.522207:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878934.522208:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878934.522210:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878934.522212:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290113280:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878934.522217:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878934.522219:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878934.522220:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878934.522221:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878934.522224:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.522228:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878934.522229:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:1.0:1788878934.522232:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878934.522241:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878934.522244:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878934.522246:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878934.522247:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.522249:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878934.522251:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.522254:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304545802 00000100:00000001:1.0:1788878934.522256:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304546345 00000100:00000001:1.0:1788878934.522258:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304545802 00000100:00000001:1.0:1788878934.522260:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304546345 00000100:00000001:1.0:1788878934.522262:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878934.522263:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.522265:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878934.522268:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878934/real 1788878934] req@ffff94e5045e6680 x1875765290113280/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878934.522277:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878934.522277:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878934.522279:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878934.522298:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878934.522298:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.522300:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878934.522301:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878934.522304:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290113280/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878934.522317:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.522320:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878934.522322:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.522323:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878934.522325:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.522327:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878934.522352:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878934.522356:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878934.522371:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878934.522389:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878934.522393:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290113280:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878934.522398:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.522400:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878934.522402:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878934.522403:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878934.522406:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878934.522407:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878934.522409:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878934.522411:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878934.522412:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878934.522414:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878934.522416:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.522417:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878934.522418:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878934.522419:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878934.522421:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878934.522422:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878934.522423:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878934.522424:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878934.522425:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878934.522427:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878934.522428:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.522429:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878934.522430:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878934.522432:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878934.522433:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878934.522434:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878934.522436:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878934.522437:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878934.522438:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878934.522439:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878934.522440:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.522442:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878934.522445:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878934.522447:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878934.522448:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878934.522450:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878934.584993:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878934.584999:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878934.585000:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878934.585004:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878934.585007:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878934.585009:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878934.585011:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878934.585013:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878934.585015:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878935.289007:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878935.289015:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878935.289017:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878935.289020:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878935.289025:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878935.289027:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878935.289029:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878935.289031:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878935.289034:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878935.289036:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878935.289055:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878935.289057:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878935.289060:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878935.608995:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878935.609002:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878935.609004:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878935.609009:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878935.609013:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878935.609016:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878935.609019:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878935.609021:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878935.609023:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878936.312968:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878936.312973:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878936.312978:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878936.312979:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878936.312995:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878936.312997:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878936.312999:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878936.632982:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878936.632990:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878936.632992:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878936.632998:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878936.633001:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878936.633005:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878936.633008:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878936.633010:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878936.633013:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878937.336977:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878937.336982:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878937.336986:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878937.336987:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878937.337045:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878937.337048:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878937.337051:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878937.657008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878937.657016:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878937.657018:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878937.657024:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878937.657027:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878937.657030:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878937.657032:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878937.657034:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878937.657036:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878938.361074:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878938.361082:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878938.361087:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878938.361088:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878938.361154:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878938.361172:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878938.361177:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878938.681011:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878938.681021:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878938.681023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878938.681028:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878938.681032:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878938.681037:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878938.681040:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878938.681042:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878938.681045:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878939.385106:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878939.385115:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878939.385121:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878939.385123:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878939.385225:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878939.385230:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878939.385236:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878939.641244:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878939.641264:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878939.641269:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878939.641277:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878939.641284:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878939.641295:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9878 00000100:00080000:0.0:1788878939.641336:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878939.641357:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878939.641360:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878939.641373:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878939.641377:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878939.641381:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878939.641384:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878939.641387:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878939.641388:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878939.641394:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878939.641396:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878939.641400:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878939.641405:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878939.641409:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.641413:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878939.641417:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878939.641436:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878939.641438:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.641443:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878939.641447:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878939.641472:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.641496:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878939.641515:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979180 x1875765290113408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878939.641536:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878939.641544:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.641596:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878939.641600:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878939.641605:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878939.641606:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878939.641609:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878939.641612:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878939.641616:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9878 00000100:00080000:0.0:1788878939.641621:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878939.641624:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878939.641626:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878939.641630:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878939.641632:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878939.641634:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788878939.641637:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878939.641640:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878939.641642:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878939.641646:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878939.641649:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878939.641652:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878939.641660:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878939.641662:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.641664:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878939.641670:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878939.641670:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:0.0:1788878939.641678:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.641680:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878939.641681:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.641687:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1788878939.641691:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788878939.641692:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788878939.641695:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.641697:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.641700:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878939.641705:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878939.641709:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878939.641709:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878939.641716:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878939.641720:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878939.641722:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878939.641737:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290113536/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788878939.641737:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290113408:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878939.641755:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788878939.641760:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788878939.641764:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788878939.641765:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878939.641770:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.641772:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878939.641780:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1788878939.641781:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878939.641788:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878939.641789:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878939.641791:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1788878939.641792:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.641795:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878939.641801:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9878 00000100:00080000:0.0:1788878939.641804:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878939.641805:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878939.641807:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878939.641810:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878939.641811:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878939.641814:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788878939.641816:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878939.641818:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878939.641819:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878939.641821:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1788878939.641823:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878939.641827:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 00000100:00000001:0.0:1788878939.641829:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878939.641834:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878939.641838:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1788878939.641839:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878939.641845:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.641848:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878939.641850:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878939.641852:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.641854:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878939.641859:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878939.641860:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.641879:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878939.641881:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878939.641884:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.641890:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878939.641895:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290113664/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878939.641905:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878939.641908:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.641911:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878939.641914:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878939.641991:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878939.642010:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878939.642016:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788878939.642019:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878939.642022:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878939.642030:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9878 00000100:00080000:0.0:1788878939.642037:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878939.642040:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878939.642042:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878939.642047:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:0.0:1788878939.642048:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878939.642056:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878939.642060:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788878939.642061:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1788878939.642064:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878939.642086:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878939.642088:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878939.642092:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878939.642101:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.642102:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878939.642109:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878939.642112:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:0.0:1788878939.642115:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878939.642119:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.642122:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878939.642126:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878939.642127:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788878939.642133:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.642139:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642140:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304551466 02000000:00000001:0.0:1788878939.642147:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878939.642152:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.642156:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878939.642160:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788878939.642160:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1788878939.642169:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642178:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304551466 00000100:00000001:0.0:1788878939.642184:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642189:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00100000:0.0:1788878939.642199:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290113792/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878939.642201:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878939.642210:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642220:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788878939.642230:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878939.642236:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878939.642244:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878939/real 1788878939] req@ffff94e508979180 x1875765290113408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878939.642245:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788878939.642267:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878939.642270:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878939.642276:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878939.642279:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878939.642282:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642286:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878939.642289:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878939.642296:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290113408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878939.642326:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642333:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878939.642354:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642357:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878939.642359:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642363:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878939.642367:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878939.642371:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878939.642379:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878939.642384:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878939.642391:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290113408:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878939.642400:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642405:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878939.642408:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878939.642412:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878939.642418:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878939.642421:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878939.642424:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878939.642429:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878939.642432:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878939.642436:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878939.642438:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642441:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642450:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878939.642469:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642472:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878939.642474:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878939.642476:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878939.642478:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878939.642480:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878939.642483:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878939.642485:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878939.642487:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878939.642492:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290113536:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878939.642496:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878939.642498:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878939.642500:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878939.642503:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878939.642506:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642515:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878939.642516:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000100:00000001:1.0:1788878939.642523:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878939.642540:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878939.642545:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878939.642548:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878939.642550:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642552:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878939.642555:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642560:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304550924 00000100:00000001:1.0:1788878939.642564:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304551466 00000100:00000001:1.0:1788878939.642567:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304550924 00000100:00000001:1.0:1788878939.642570:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304551466 00000100:00000001:1.0:1788878939.642573:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878939.642592:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642594:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878939.642598:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878939/real 1788878939] req@ffff94e50897b800 x1875765290113536/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878939.642607:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878939.642608:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878939.642611:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878939.642612:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878939.642614:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642616:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878939.642618:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878939.642622:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290113536/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878939.642629:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642631:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878939.642634:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642636:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878939.642638:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642640:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878939.642660:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878939.642662:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878939.642665:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878939.642668:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878939.642671:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290113536:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878939.642675:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878939.642676:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878939.642679:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878939.642681:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878939.642683:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878939.642685:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878939.642688:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290113664:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878939.642691:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878939.642692:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878939.642694:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878939.642695:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878939.642698:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642704:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878939.642705:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 00000100:00000001:1.0:1788878939.642709:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878939.642719:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878939.642723:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878939.642725:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878939.642726:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642728:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878939.642730:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642734:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304550922 00000100:00000001:1.0:1788878939.642737:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304551466 00000100:00000001:1.0:1788878939.642739:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304550922 00000100:00000001:1.0:1788878939.642741:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304551466 00000100:00000001:1.0:1788878939.642744:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878939.642745:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642747:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878939.642750:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878939/real 1788878939] req@ffff94e508978380 x1875765290113664/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878939.642758:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878939.642759:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878939.642761:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878939.642762:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878939.642763:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642766:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878939.642768:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878939.642771:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290113664/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878939.642779:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642781:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878939.642783:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642784:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878939.642786:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642787:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878939.642804:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878939.642806:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878939.642809:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878939.642811:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878939.642814:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290113664:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878939.642817:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878939.642818:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878939.642819:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878939.642822:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878939.642823:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878939.642826:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878939.642829:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290113792:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878939.642831:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878939.642833:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878939.642834:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878939.642836:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878939.642838:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642842:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878939.642844:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:1.0:1788878939.642847:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878939.642855:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878939.642877:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878939.642879:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878939.642881:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642883:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878939.642886:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642889:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304550922 00000100:00000001:1.0:1788878939.642892:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304551466 00000100:00000001:1.0:1788878939.642895:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304550922 00000100:00000001:1.0:1788878939.642897:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304551466 00000100:00000001:1.0:1788878939.642900:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878939.642902:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.642904:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878939.642908:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878939/real 1788878939] req@ffff94e508979c00 x1875765290113792/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878939.642916:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878939.642917:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878939.642919:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878939.642920:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878939.642921:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642923:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878939.642925:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878939.642928:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290113792/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878939.642935:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642937:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878939.642939:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642941:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878939.642943:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.642945:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878939.642961:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878939.642994:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878939.642999:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878939.643001:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878939.643005:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290113792:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878939.643009:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.643012:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878939.643013:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878939.643015:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878939.643018:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878939.643019:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878939.643020:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878939.643024:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878939.643025:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878939.643029:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878939.643030:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.643032:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878939.643034:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878939.643035:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878939.643037:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878939.643039:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878939.643040:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878939.643041:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878939.643042:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878939.643044:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878939.643045:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.643047:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878939.643048:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878939.643049:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878939.643051:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878939.643052:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878939.643053:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878939.643055:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878939.643056:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878939.643058:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878939.643059:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.643061:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878939.643064:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878939.643066:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878939.643068:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878939.643069:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878939.705024:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878939.705035:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878939.705039:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878939.705048:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878939.705054:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878939.705060:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878939.705065:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878939.705069:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878939.705074:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878940.409092:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878940.409099:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878940.409105:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878940.409107:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878940.409133:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878940.409136:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878940.409140:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878940.728975:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878940.728981:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878940.728982:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878940.728986:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878940.728989:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878940.728992:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878940.728994:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878940.728996:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878940.728998:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878941.432991:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878941.432996:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878941.433000:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878941.433001:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878941.433053:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878941.433057:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878941.433061:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878941.752980:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878941.752988:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878941.752990:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878941.752997:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878941.753000:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878941.753004:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878941.753007:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878941.753009:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878941.753013:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878942.456985:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878942.456991:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878942.456996:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878942.456998:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878942.457000:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878942.457003:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878942.457005:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878942.457006:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878942.457007:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878942.457009:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878942.457093:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878942.457098:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878942.457101:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878942.777047:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878942.777055:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878942.777072:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878942.777078:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878942.777082:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878942.777086:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878942.777089:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878942.777092:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878942.777095:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878943.481044:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878943.481052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878943.481069:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878943.481070:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878943.481097:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878943.481100:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878943.481103:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878943.801027:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878943.801035:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878943.801038:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878943.801044:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878943.801048:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878943.801052:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878943.801055:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878943.801057:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878943.801060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878944.505029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878944.505036:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878944.505042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878944.505043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878944.505065:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878944.505068:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878944.505071:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878944.761008:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878944.761019:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878944.761021:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878944.761025:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878944.761029:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878944.761037:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9883 00000100:00080000:0.0:1788878944.761047:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878944.761051:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878944.761053:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878944.761062:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878944.761064:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878944.761068:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878944.761071:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878944.761073:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878944.761074:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878944.761079:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878944.761082:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878944.761085:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878944.761089:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878944.761092:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878944.761095:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878944.761098:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878944.761109:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878944.761111:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878944.761115:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878944.761118:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878944.761123:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878944.761135:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878944.761146:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290113920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878944.761157:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878944.761161:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878944.761178:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878944.761182:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878944.761186:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878944.761188:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878944.761190:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878944.761193:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878944.761196:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9883 00000100:00080000:0.0:1788878944.761201:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878944.761204:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878944.761206:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878944.761210:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878944.761212:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878944.761214:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788878944.761217:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788878944.761218:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878944.761223:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878944.761223:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878944.761227:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878944.761227:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878944.761232:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788878944.761235:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878944.761237:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878944.761241:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878944.761244:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878944.761245:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788878944.761249:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878944.761254:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878944.761259:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788878944.761261:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290113920:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788878944.761266:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761280:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788878944.761285:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788878944.761286:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:0.0:1788878944.761290:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788878944.761291:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878944.761295:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:0.0:1788878944.761297:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878944.761301:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878944.761302:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878944.761307:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878944.761310:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878944.761313:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878944.761315:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878944.761317:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761322:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878944.761324:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761326:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:1.0:1788878944.761332:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1788878944.761334:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290114048/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878944.761345:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878944.761347:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878944.761351:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878944.761355:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878944.761359:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878944.761360:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878944.761362:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878944.761366:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000800:00000100:1.0:1788878944.761370:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1788878944.761373:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9883 00000100:00080000:0.0:1788878944.761379:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878944.761382:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878944.761384:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878944.761385:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878944.761391:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1788878944.761392:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788878944.761393:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878944.761395:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788878944.761397:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:0.0:1788878944.761400:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788878944.761402:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878944.761406:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788878944.761407:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304556585 00000100:00000001:0.0:1788878944.761412:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878944.761415:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878944.761417:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1788878944.761421:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878944.761426:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761426:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304556585 02000000:00000001:0.0:1788878944.761432:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878944.761434:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1788878944.761436:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878944.761441:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761441:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878944.761443:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788878944.761446:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788878944.761448:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878944.761451:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000008:00000001:0.0:1788878944.761453:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878944.761457:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878944.761461:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878944/real 1788878944] req@ffff94e50897ad80 x1875765290113920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788878944.761463:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878944.761466:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878944.761471:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878944.761474:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878944.761477:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878944.761485:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788878944.761496:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:0.0:1788878944.761497:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290114176/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1788878944.761500:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878944.761506:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878944.761509:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878944.761511:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878944.761515:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761516:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878944.761520:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761522:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878944.761525:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788878944.761526:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788878944.761531:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290113920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788878944.761533:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878944.761537:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878944.761541:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878944.761542:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878944.761547:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788878944.761548:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878944.761552:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788878944.761553:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9883 00000100:00000001:1.0:1788878944.761554:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878944.761556:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.761560:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1788878944.761561:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878944.761562:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878944.761568:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00080000:1.0:1788878944.761568:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1788878944.761572:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788878944.761575:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878944.761579:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:0.0:1788878944.761580:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:1.0:1788878944.761583:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290113920:192.168.203.129@tcp:250:kworker.0 00000020:00000001:0.0:1788878944.761584:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788878944.761590:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.761592:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878944.761593:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878944.761594:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878944.761597:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878944.761598:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:1.0:1788878944.761599:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878944.761601:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000020:00000001:0.0:1788878944.761605:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788878944.761606:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878944.761610:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878944.761611:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878944.761614:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878944.761615:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878944.761617:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.761620:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788878944.761623:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878944.761628:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878944.761629:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761631:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878944.761635:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878944.761635:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878944.761640:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878944.761641:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1788878944.761642:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878944.761643:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878944.761646:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878944.761649:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878944.761649:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878944.761651:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878944.761654:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788878944.761655:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788878944.761660:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788878944.761661:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290114048:192.168.203.129@tcp:38:kworker.0 00000008:00000001:0.0:1788878944.761668:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761668:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878944.761670:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788878944.761674:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878944.761674:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878944.761678:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878944.761682:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878944.761683:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878944.761686:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761688:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878944.761689:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788878944.761692:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 02000000:00000001:0.0:1788878944.761693:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788878944.761697:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1788878944.761698:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878944.761706:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788878944.761710:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:0.0:1788878944.761715:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290114304/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878944.761716:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878944.761719:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878944.761722:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761726:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878944.761729:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761734:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304556043 00000100:00000001:0.0:1788878944.761735:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878944.761739:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304556585 00000100:00000001:0.0:1788878944.761741:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761742:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304556043 00000100:00000001:1.0:1788878944.761746:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304556585 00000100:00000001:0.0:1788878944.761749:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761749:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878944.761754:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761756:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878944.761760:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878944/real 1788878944] req@ffff94e50897bb80 x1875765290114048/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878944.761771:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878944.761772:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878944.761774:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878944.761775:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878944.761776:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.761777:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878944.761778:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878944.761781:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290114048/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878944.761786:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.761788:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878944.761790:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.761791:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878944.761792:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.761793:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878944.761803:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878944.761804:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878944.761806:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878944.761807:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878944.761811:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290114048:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878944.761814:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878944.761815:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878944.761817:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878944.761818:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878944.761819:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878944.761820:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878944.761823:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290114176:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878944.761825:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878944.761826:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878944.761828:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878944.761829:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878944.761832:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761835:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878944.761836:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:1.0:1788878944.761838:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878944.761847:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878944.761851:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878944.761854:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878944.761855:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761856:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878944.761858:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761860:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304556041 00000100:00000001:1.0:1788878944.761862:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304556585 00000100:00000001:1.0:1788878944.761864:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304556041 00000100:00000001:1.0:1788878944.761865:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304556585 00000100:00000001:1.0:1788878944.761867:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878944.761868:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.761869:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878944.761872:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878944/real 1788878944] req@ffff94e508978e00 x1875765290114176/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878944.761877:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878944.761878:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878944.761880:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878944.761882:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878944.761883:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.761884:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878944.761885:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878944.761888:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290114176/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878944.761893:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.761895:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878944.761897:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.761899:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878944.761900:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.761931:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878944.761949:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878944.761952:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878944.761957:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878944.761959:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878944.761962:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290114176:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878944.761965:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.761967:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878944.761969:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878944.761971:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878944.761974:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878944.761975:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878944.761977:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878944.761979:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878944.761981:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878944.761983:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878944.761985:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.761986:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878944.761988:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878944.761991:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878944.761993:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878944.761993:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878944.761994:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878944.761997:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878944.761999:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878944.762001:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878944.762003:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.762004:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.762007:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878944.762008:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.762011:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878944.762012:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878944.762013:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878944.762015:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878944.762017:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878944.762018:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878944.762020:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878944.762021:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878944.762023:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290114304:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878944.762025:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878944.762027:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878944.762028:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878944.762030:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878944.762032:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.762037:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878944.762039:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:1.0:1788878944.762042:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878944.762051:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878944.762056:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878944.762058:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878944.762059:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.762060:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878944.762062:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.762065:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304556042 00000100:00000001:1.0:1788878944.762067:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304556585 00000100:00000001:1.0:1788878944.762069:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304556042 00000100:00000001:1.0:1788878944.762071:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304556585 00000100:00000001:1.0:1788878944.762073:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878944.762074:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.762076:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878944.762084:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878944/real 1788878944] req@ffff94e508978a80 x1875765290114304/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878944.762089:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878944.762090:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878944.762093:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878944.762095:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878944.762096:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.762098:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878944.762100:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878944.762103:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290114304/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878944.762108:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.762111:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878944.762113:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.762114:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878944.762115:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.762117:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878944.762130:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878944.762132:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878944.762135:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878944.762137:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878944.762139:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290114304:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878944.762141:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.762142:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878944.762143:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878944.762144:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878944.762146:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878944.762146:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878944.762147:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878944.762149:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878944.762150:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878944.762151:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878944.762152:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.762153:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878944.762154:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878944.762156:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878944.762157:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878944.762158:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878944.825008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878944.825013:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878944.825014:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878944.825018:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878944.825021:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878944.825023:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878944.825025:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878944.825028:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878944.825029:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878945.529090:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878945.529100:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878945.529103:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878945.529107:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878945.529113:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878945.529116:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878945.529119:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878945.529122:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878945.529126:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878945.529128:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878945.529212:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878945.529218:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878945.529223:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878945.849032:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878945.849041:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878945.849044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878945.849050:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878945.849055:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878945.849059:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878945.849063:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878945.849066:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878945.849069:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878946.553031:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878946.553039:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878946.553045:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878946.553047:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878946.553127:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878946.553133:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878946.553137:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878946.873065:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878946.873074:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878946.873078:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878946.873085:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878946.873092:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878946.873098:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878946.873102:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878946.873106:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878946.873110:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878947.577036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878947.577042:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878947.577048:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878947.577050:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878947.577076:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878947.577078:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878947.577081:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878947.897008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878947.897017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878947.897020:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878947.897027:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878947.897032:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878947.897037:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878947.897040:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878947.897043:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878947.897047:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878948.601055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878948.601062:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878948.601067:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878948.601068:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878948.601089:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878948.601091:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878948.601094:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878948.921092:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878948.921099:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878948.921101:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878948.921106:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878948.921110:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878948.921114:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878948.921117:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878948.921120:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878948.921123:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878949.625136:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878949.625146:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878949.625154:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878949.625157:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878949.625317:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878949.625325:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878949.625332:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878949.881047:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878949.881060:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878949.881064:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878949.881069:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878949.881075:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878949.881084:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9888 00000100:00080000:0.0:1788878949.881097:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878949.881101:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878949.881103:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878949.881114:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878949.881117:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878949.881131:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878949.881134:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878949.881140:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878949.881143:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878949.881148:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878949.881151:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878949.881155:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878949.881160:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878949.881162:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881165:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878949.881168:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878949.881183:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878949.881185:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881188:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878949.881192:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878949.881197:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881225:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878949.881256:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290114432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878949.881270:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878949.881275:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881292:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878949.881295:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878949.881299:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878949.881301:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878949.881303:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878949.881306:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878949.881308:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9888 00000100:00080000:0.0:1788878949.881312:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878949.881314:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878949.881316:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878949.881319:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878949.881320:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878949.881322:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788878949.881325:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788878949.881328:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878949.881329:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878949.881333:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878949.881336:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878949.881338:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878949.881345:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878949.881347:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881349:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878949.881353:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878949.881356:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881359:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.881360:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788878949.881363:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878949.881368:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881371:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788878949.881371:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788878949.881377:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788878949.881379:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1788878949.881380:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.881386:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878949.881390:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.881392:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878949.881399:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897aa00 x1875765290114560/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788878949.881403:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878949.881410:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878949.881412:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878949.881417:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878949.881420:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881422:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878949.881428:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878949.881433:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878949.881435:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788878949.881435:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290114432:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1788878949.881438:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878949.881443:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878949.881447:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9888 00000100:00080000:0.0:1788878949.881450:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878949.881451:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878949.881453:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878949.881461:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1788878949.881462:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878949.881467:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878949.881469:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 02000000:00000001:1.0:1788878949.881471:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1788878949.881472:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878949.881478:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788878949.881479:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881480:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878949.881489:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788878949.881489:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788878949.881495:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878949.881496:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878949.881499:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878949.881501:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878949.881501:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881508:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878949.881511:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878949.881513:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881515:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878949.881520:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878949.881521:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881522:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878949.881523:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878949.881525:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881531:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.881532:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00100000:0.0:1788878949.881537:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978700 x1875765290114688/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878949.881538:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:1.0:1788878949.881550:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788878949.881551:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878949.881555:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881558:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878949.881561:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878949.881565:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878949.881567:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878949.881568:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878949.881571:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878949.881574:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9888 00000100:00080000:0.0:1788878949.881595:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878949.881597:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878949.881598:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878949.881601:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878949.881602:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878949.881604:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788878949.881606:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878949.881608:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878949.881609:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878949.881612:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878949.881615:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878949.881616:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878949.881619:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878949.881620:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881622:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878949.881623:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788878949.881625:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881627:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878949.881631:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878949.881633:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.881636:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788878949.881636:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878949.881640:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878949.881644:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.881648:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304561705 00000100:00000001:0.0:1788878949.881650:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878949.881658:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290114816/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878949.881665:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878949.881668:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.881668:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878949.881672:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.881683:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304561705 00000100:00000001:1.0:1788878949.881693:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788878949.881702:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878949.881709:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.881715:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.881720:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878949.881731:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878949.881735:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788878949.881740:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878949.881745:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878949.881748:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878949.881750:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878949.881752:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878949.881757:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878949.881761:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878949.881764:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878949.881772:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290114560:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878949.881778:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878949.881781:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878949.881784:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878949.881787:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878949.881792:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.881803:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878949.881805:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:1.0:1788878949.881812:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788878949.881838:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.881846:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304561163 00000100:00000001:1.0:1788878949.881850:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304561705 00000100:00000001:1.0:1788878949.881855:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304561163 00000100:00000001:1.0:1788878949.881859:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304561705 00000100:00000001:1.0:1788878949.881862:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878949.881866:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.881869:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878949.881872:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878949.881875:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878949.881879:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878949.881883:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878949.881886:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878949.881893:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290114688:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878949.881899:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878949.881946:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878949.881950:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878949.881954:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878949.881959:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.881970:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878949.881973:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00000001:1.0:1788878949.881978:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788878949.881995:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.882001:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304561162 00000100:00000001:1.0:1788878949.882006:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304561705 00000100:00000001:1.0:1788878949.882010:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304561162 00000100:00000001:1.0:1788878949.882013:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304561705 00000100:00000001:1.0:1788878949.882017:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878949.882020:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.882023:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878949.882025:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878949.882027:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878949.882030:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878949.882033:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878949.882036:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878949.882043:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290114816:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878949.882047:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878949.882050:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878949.882053:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878949.882055:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878949.882060:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.882069:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878949.882071:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:1.0:1788878949.882076:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788878949.882103:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.882126:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304561162 00000100:00000001:1.0:1788878949.882131:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304561705 00000100:00000001:1.0:1788878949.882134:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304561162 00000100:00000001:1.0:1788878949.882138:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304561705 00000100:00000001:1.0:1788878949.882142:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878949.882145:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.882149:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.882152:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878949.882157:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878949.882160:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788878949.882164:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878949.882166:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878949.882168:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878949.882171:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878949.945023:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878949.945033:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878949.945036:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878949.945043:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878949.945048:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878949.945053:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878949.945056:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878949.945061:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878949.945065:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878950.649083:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878950.649095:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878950.649102:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878950.649104:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878950.649210:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878950.649217:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878950.649257:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878950.969095:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878950.969107:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878950.969111:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878950.969119:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878950.969125:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878950.969131:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878950.969152:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878950.969156:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878950.969160:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878951.673149:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878951.673158:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878951.673164:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878951.673166:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878951.673194:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878951.673198:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878951.673201:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878951.992995:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878951.993003:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878951.993005:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878951.993011:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878951.993015:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878951.993020:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878951.993024:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878951.993027:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878951.993030:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878952.696972:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878952.696977:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878952.696981:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878952.696984:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878952.696986:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878952.696988:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878952.696990:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878952.696991:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878952.696993:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878952.696994:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878952.697010:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878952.697012:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878952.697014:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788878952.953172:0:2352:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1788878952.953209:0:2352:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788878952.953239:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788878952.953251:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878952.953265:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878952.953267:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878952.953270:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788878952.953277:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788878952.953281:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878952.953286:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878952.953287:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878952.953289:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788878952.953295:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:1.0:1788878952.953298:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878952.953301:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878952.953304:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878952.953306:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878952.953310:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788878952.953311:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878952.953315:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788878952.953319:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788878952.953324:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878952.953327:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878952.953329:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878952.953332:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:1.0:1788878952.953334:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878949/real 1788878952] req@ffff94e50897aa00 x1875765290114560/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878952.953351:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878952.953354:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878952.953361:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878952.953364:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878952.953365:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953369:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878952.953371:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878952.953376:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290114560/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878952.953383:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953387:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878952.953390:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953392:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878952.953393:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953397:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878952.953412:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878952.953417:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878952.953422:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878952.953425:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878952.953429:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290114560:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878952.953435:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878952.953438:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878949/real 1788878952] req@ffff94e508978700 x1875765290114688/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878952.953445:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878952.953447:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878952.953450:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878952.953452:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878952.953454:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953471:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878952.953472:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878952.953477:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290114688/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878952.953484:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953486:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878952.953489:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953490:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878952.953492:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953493:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878952.953506:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878952.953509:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878952.953513:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878952.953516:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878952.953519:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290114688:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878952.953524:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878952.953529:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878949/real 1788878952] req@ffff94e508978a80 x1875765290114816/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878952.953538:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878952.953540:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878952.953543:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878952.953545:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878952.953547:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953548:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878952.953549:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878952.953553:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290114816/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878952.953560:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953563:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878952.953565:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953567:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878952.953568:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953570:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878952.953590:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878952.953593:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878952.953596:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878952.953598:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878952.953602:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290114816:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878952.953606:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878952.953610:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878949/real 1788878952] req@ffff94e50897a300 x1875765290114432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878952.953619:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878952.953620:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878952.953624:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878952.953625:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878952.953627:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953628:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878952.953630:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878952.953634:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290114432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878952.953640:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953643:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878952.953645:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953647:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878952.953648:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953650:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878952.953651:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878952.953653:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878952.953657:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878952.953659:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878952.953662:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290114432:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878952.953666:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953669:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878952.953670:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878952.953674:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878952.953679:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878952.953682:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878952.953684:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878952.953688:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878952.953690:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878952.953694:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878952.953696:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953698:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878952.953699:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878952.953700:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878952.953702:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878952.953703:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878952.953704:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878952.953706:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878952.953708:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878952.953709:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878952.953710:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953712:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878952.953713:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878952.953715:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878952.953716:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878952.953717:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878952.953718:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878952.953721:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878952.953722:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878952.953723:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878952.953724:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953726:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878952.953727:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878952.953728:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878952.953730:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878952.953731:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878952.953732:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878952.953734:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878952.953735:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878952.953736:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878952.953737:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953740:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878952.953748:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878952.953750:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878952.953753:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878952.953754:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878953.016981:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878953.016987:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878953.016989:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878953.016993:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878953.016996:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878953.016999:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878953.017001:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878953.017003:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878953.017005:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878953.721001:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878953.721007:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878953.721011:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878953.721012:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878953.721076:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878953.721080:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878953.721083:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788878954.040984:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878954.040992:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878954.040994:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878954.040998:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878954.041002:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878954.041006:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878954.041008:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878954.041011:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878954.041013:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878954.745060:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878954.745069:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878954.745075:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878954.745077:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878954.745167:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878954.745173:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878954.745178:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788878955.001180:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878955.001194:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788878955.001241:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878955.001245:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878955.001251:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878955.001258:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9893 00000100:00080000:2.0:1788878955.001270:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878955.001274:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878955.001276:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878955.001285:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878955.001288:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878955.001291:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788878955.001294:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788878955.001296:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878955.001298:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878955.001303:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878955.001305:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878955.001308:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878955.001312:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878955.001315:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878955.001317:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788878955.001320:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878955.001330:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878955.001331:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878955.001334:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878955.001337:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878955.001342:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878955.001354:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788878955.001364:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290114944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788878955.001375:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788878955.001380:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878955.001399:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878955.001402:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878955.001408:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788878955.001410:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878955.001412:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788878955.001415:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788878955.001418:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9893 00000100:00080000:2.0:1788878955.001421:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878955.001423:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878955.001425:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878955.001428:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878955.001430:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878955.001433:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788878955.001435:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788878955.001438:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878955.001440:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878955.001444:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788878955.001446:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878955.001447:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878955.001452:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788878955.001454:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878955.001460:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.001462:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788878955.001463:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788878955.001471:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:2.0:1788878955.001472:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788878955.001479:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.001481:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788878955.001482:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.001488:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878955.001490:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788878955.001494:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878955.001498:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788878955.001499:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788878955.001500:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878955.001505:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878955.001512:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878955.001517:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1788878955.001526:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290115072/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788878955.001527:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878955.001543:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290114944:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878955.001557:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878955.001563:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878955.001567:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878955.001576:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788878955.001595:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788878955.001597:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878955.001600:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878955.001609:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788878955.001614:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878955.001620:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788878955.001623:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788878955.001624:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788878955.001627:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878955.001631:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:2.0:1788878955.001636:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788878955.001641:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788878955.001645:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9893 00000100:00080000:2.0:1788878955.001656:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878955.001660:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878955.001662:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878955.001669:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878955.001671:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878955.001674:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788878955.001679:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788878955.001683:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878955.001686:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788878955.001692:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:1.0:1788878955.001709:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788878955.001712:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878955.001719:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878955.001725:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878955.001728:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:2.0:1788878955.001730:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.001737:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878955.001737:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788878955.001745:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788878955.001747:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878955.001753:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000008:00000001:2.0:1788878955.001756:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878955.001763:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.001766:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788878955.001771:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788878955.001776:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304566825 02000000:00000001:2.0:1788878955.001777:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878955.001785:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878955.001788:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878955.001792:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.001795:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788878955.001803:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.001807:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304566825 00000100:00100000:2.0:1788878955.001818:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290115200/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878955.001821:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788878955.001831:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878955.001841:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878955.001845:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878955.001848:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788878955.001851:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878955.001859:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878955.001865:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878955/real 1788878955] req@ffff94e5045e4700 x1875765290114944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788878955.001866:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788878955.001882:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788878955.001888:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788878955.001893:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:1.0:1788878955.001945:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878955.001951:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878955.001959:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878955.001963:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788878955.001967:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788878955.001968:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878955.001978:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9893 00000100:00000001:1.0:1788878955.001979:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878955.001985:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788878955.001993:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788878955.002001:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1788878955.002002:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290114944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788878955.002004:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788878955.002015:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788878955.002020:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788878955.002025:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788878955.002030:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788878955.002033:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788878955.002040:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878955.002041:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788878955.002046:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878955.002051:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.002059:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1788878955.002060:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878955.002066:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788878955.002069:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002074:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1788878955.002077:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788878955.002085:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788878955.002088:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002090:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788878955.002092:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788878955.002098:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1788878955.002102:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000008:00000001:2.0:1788878955.002107:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788878955.002115:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788878955.002116:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 02000000:00000001:2.0:1788878955.002125:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788878955.002126:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1788878955.002132:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878955.002139:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290114944:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788878955.002141:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788878955.002147:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788878955.002153:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002157:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.002164:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788878955.002165:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002167:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878955.002178:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:2.0:1788878955.002185:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290115328/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788878955.002187:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878955.002193:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878955.002220:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878955.002226:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1788878955.002229:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788878955.002232:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788878955.002238:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002241:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788878955.002246:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002253:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.002259:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.002272:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878955.002276:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002280:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878955.002283:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878955.002285:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878955.002288:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878955.002291:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878955.002296:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878955.002299:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878955.002302:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878955.002309:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290115072:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878955.002315:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878955.002318:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878955.002321:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878955.002323:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878955.002329:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002339:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878955.002342:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00000001:1.0:1788878955.002348:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878955.002371:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878955.002379:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878955.002383:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878955.002385:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002389:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878955.002393:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002400:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304566284 00000100:00000001:1.0:1788878955.002404:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304566825 00000100:00000001:1.0:1788878955.002407:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304566284 00000100:00000001:1.0:1788878955.002411:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304566825 00000100:00000001:1.0:1788878955.002414:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878955.002416:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002419:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878955.002425:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878955/real 1788878955] req@ffff94e5045e7480 x1875765290115072/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878955.002438:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878955.002440:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878955.002444:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878955.002447:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878955.002449:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.002451:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878955.002453:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878955.002459:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290115072/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878955.002469:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.002472:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878955.002477:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.002479:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878955.002481:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.002484:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878955.002503:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878955.002508:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878955.002514:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878955.002518:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878955.002524:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290115072:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878955.002529:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878955.002531:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878955.002533:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878955.002536:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878955.002539:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878955.002542:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878955.002547:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290115200:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878955.002552:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878955.002555:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878955.002557:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878955.002560:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878955.002564:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002573:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878955.002575:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00000001:1.0:1788878955.002580:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878955.002599:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878955.002606:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878955.002609:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878955.002620:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002623:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878955.002628:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002634:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304566282 00000100:00000001:1.0:1788878955.002638:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304566825 00000100:00000001:1.0:1788878955.002641:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304566282 00000100:00000001:1.0:1788878955.002645:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304566825 00000100:00000001:1.0:1788878955.002648:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878955.002651:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002654:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878955.002661:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878955/real 1788878955] req@ffff94e5045e7b80 x1875765290115200/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878955.002681:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878955.002683:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878955.002688:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878955.002690:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878955.002692:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.002695:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878955.002697:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878955.002703:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290115200/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878955.002712:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.002715:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878955.002718:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.002721:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878955.002723:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.002726:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878955.002745:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878955.002749:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878955.002755:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878955.002759:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878955.002764:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290115200:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878955.002769:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878955.002772:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878955.002774:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878955.002796:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878955.002799:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878955.002802:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878955.002807:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290115328:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878955.002812:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878955.002815:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878955.002818:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878955.002820:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878955.002825:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002833:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878955.002835:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:1.0:1788878955.002839:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878955.002870:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878955.002877:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878955.002881:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878955.002884:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002887:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878955.002890:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002897:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304566282 00000100:00000001:1.0:1788878955.002900:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304566825 00000100:00000001:1.0:1788878955.002937:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304566283 00000100:00000001:1.0:1788878955.002941:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304566825 00000100:00000001:1.0:1788878955.002944:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878955.002948:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.002952:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878955.002959:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878955/real 1788878955] req@ffff94e5045e4e00 x1875765290115328/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878955.002974:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878955.002976:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878955.002981:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878955.002984:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878955.002986:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.002989:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878955.002992:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878955.003007:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290115328/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878955.003018:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.003022:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878955.003026:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.003029:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878955.003031:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.003034:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878955.003051:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878955.003056:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878955.003061:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878955.003065:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878955.003070:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290115328:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878955.003076:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.003079:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878955.003081:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878955.003083:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878955.003088:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878955.003090:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878955.003093:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878955.003096:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878955.003098:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878955.003101:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878955.003103:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.003105:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878955.003107:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878955.003108:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878955.003110:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878955.003112:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878955.003113:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878955.003114:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878955.003115:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878955.003117:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878955.003118:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.003119:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878955.003121:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878955.003123:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878955.003124:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878955.003126:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878955.003128:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878955.003130:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878955.003133:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878955.003136:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878955.003138:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.003141:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878955.003149:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878955.003151:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878955.003154:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878955.003157:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878955.064988:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788878955.064995:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878955.064998:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878955.065003:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788878955.065007:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878955.065011:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878955.065014:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878955.065017:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878955.065019:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878955.769040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878955.769049:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878955.769052:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878955.769055:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878955.769060:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878955.769062:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878955.769064:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878955.769067:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878955.769071:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878955.769073:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878955.769095:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878955.769097:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878955.769100:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878956.089044:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878956.089054:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878956.089057:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878956.089064:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878956.089068:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878956.089071:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878956.089074:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878956.089076:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878956.089079:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878956.793041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878956.793047:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878956.793051:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878956.793052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878956.793067:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878956.793070:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878956.793072:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878957.113011:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878957.113020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878957.113022:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878957.113028:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878957.113032:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878957.113036:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878957.113039:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878957.113041:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878957.113043:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878957.816991:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878957.816997:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878957.817001:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878957.817002:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878957.817093:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878957.817098:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878957.817102:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878958.136992:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878958.136998:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878958.137000:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878958.137021:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878958.137024:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878958.137028:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878958.137030:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878958.137033:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878958.137036:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878958.841022:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878958.841031:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878958.841038:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878958.841040:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878958.841099:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878958.841103:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878958.841107:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878959.161010:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878959.161019:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878959.161021:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878959.161028:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878959.161032:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878959.161037:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878959.161039:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878959.161042:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878959.161046:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878959.865065:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878959.865076:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878959.865084:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878959.865087:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878959.865119:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878959.865124:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878959.865129:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878960.121070:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878960.121081:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878960.121084:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878960.121088:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878960.121094:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878960.121102:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9898 00000100:00080000:3.0:1788878960.121115:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878960.121119:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878960.121121:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878960.121130:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878960.121133:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878960.121137:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878960.121141:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878960.121145:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878960.121147:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878960.121156:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878960.121177:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878960.121183:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878960.121189:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878960.121194:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878960.121198:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878960.121202:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878960.121218:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878960.121221:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878960.121226:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878960.121230:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878960.121237:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878960.121254:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878960.121270:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290115456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878960.121288:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878960.121295:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878960.121323:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878960.121337:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878960.121344:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878960.121347:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878960.121351:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878960.121356:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878960.121363:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9898 00000100:00080000:3.0:1788878960.121372:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878960.121377:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878960.121380:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878960.121387:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788878960.121388:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878960.121395:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788878960.121396:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788878960.121401:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788878960.121402:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878960.121407:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1788878960.121411:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788878960.121412:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878960.121420:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788878960.121422:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788878960.121425:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878960.121431:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788878960.121432:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878960.121444:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290115456:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1788878960.121450:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878960.121453:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878960.121461:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878960.121463:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878960.121466:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878960.121472:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788878960.121473:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878960.121483:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878960.121487:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878960.121495:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878960.121499:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788878960.121503:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000008:00100000:3.0:1788878960.121506:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878960.121509:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:1.0:1788878960.121518:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:3.0:1788878960.121520:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878960.121528:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878960.121538:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878960.121541:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878960.121545:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878960.121547:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878960.121552:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878960.121564:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788878960.121565:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1788878960.121576:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290115584/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878960.121583:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878960.121591:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878960.121592:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878960.121596:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878960.121598:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.121602:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878960.121608:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.121611:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878960.121615:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788878960.121618:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304571945 00000100:00080000:3.0:1788878960.121621:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878960.121626:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878960.121629:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878960.121633:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788878960.121634:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788878960.121638:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9898 00000100:00000001:1.0:1788878960.121646:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304571945 00000100:00080000:3.0:1788878960.121648:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878960.121655:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878960.121656:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878960.121660:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878960.121664:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788878960.121669:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788878960.121671:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.121679:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1788878960.121679:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878960.121688:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00100000:1.0:1788878960.121693:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878960/real 1788878960] req@ffff94e50449e300 x1875765290115456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788878960.121696:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878960.121708:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878960.121714:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:1.0:1788878960.121718:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878960.121723:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878960.121727:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:3.0:1788878960.121729:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878960.121732:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878960.121738:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878960.121743:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.121744:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878960.121750:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878960.121752:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878960.121762:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788878960.121780:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290115456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788878960.121784:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878960.121795:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878960.121802:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878960.121814:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.121817:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878960.121821:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.121827:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878960.121833:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788878960.121837:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878960.121840:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.121847:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788878960.121850:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788878960.121855:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878960.121857:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788878960.121865:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:3.0:1788878960.121866:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.121871:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878960.121878:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788878960.121878:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788878960.121890:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1788878960.121896:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290115712/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878960.121908:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878960.121911:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878960.121961:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.121964:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:3.0:1788878960.121969:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788878960.121977:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290115456:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1788878960.121980:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878960.121988:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788878960.121991:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788878960.121992:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878960.122001:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878960.122003:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788878960.122006:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878960.122011:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:3.0:1788878960.122012:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9898 02000000:00000001:1.0:1788878960.122018:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:3.0:1788878960.122021:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000020:00000001:1.0:1788878960.122024:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878960.122031:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788878960.122032:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878960.122037:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878960.122042:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878960.122045:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878960.122048:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788878960.122050:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1788878960.122053:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788878960.122056:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788878960.122062:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878960.122064:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788878960.122066:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878960.122069:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122075:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788878960.122077:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878960.122083:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.122086:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1788878960.122087:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878960.122093:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878960.122096:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878960.122098:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878960.122104:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788878960.122106:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.122108:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878960.122112:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788878960.122114:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878960.122119:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788878960.122119:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788878960.122128:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1788878960.122130:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878960.122133:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878960.122136:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878960.122140:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878960.122145:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1788878960.122149:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290115584:192.168.203.129@tcp:38:kworker.0 02000000:00000001:3.0:1788878960.122151:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878960.122159:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788878960.122186:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788878960.122190:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788878960.122192:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788878960.122215:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878960.122216:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878960.122225:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878960.122230:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878960.122231:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878960.122246:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290115840/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878960.122247:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878960.122254:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:3.0:1788878960.122266:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878960.122269:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878960.122270:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878960.122279:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788878960.122306:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878960.122317:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878960.122322:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878960.122325:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.122328:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878960.122333:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.122341:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304571404 00000100:00000001:1.0:1788878960.122346:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304571945 00000100:00000001:1.0:1788878960.122350:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304571404 00000100:00000001:1.0:1788878960.122354:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304571945 00000100:00000001:1.0:1788878960.122357:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878960.122361:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.122365:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878960.122374:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878960/real 1788878960] req@ffff94e50449c700 x1875765290115584/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878960.122392:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878960.122394:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878960.122399:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878960.122402:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878960.122405:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122409:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878960.122412:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878960.122420:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290115584/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878960.122434:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122437:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878960.122441:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122443:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878960.122444:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122447:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878960.122461:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878960.122464:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878960.122468:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878960.122471:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878960.122475:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290115584:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878960.122480:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878960.122481:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878960.122483:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878960.122486:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878960.122488:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878960.122491:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878960.122494:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290115712:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878960.122502:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878960.122504:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878960.122506:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878960.122508:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878960.122512:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.122520:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878960.122522:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:1.0:1788878960.122526:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878960.122542:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878960.122548:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878960.122551:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878960.122552:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.122554:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878960.122558:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.122562:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304571402 00000100:00000001:1.0:1788878960.122565:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304571945 00000100:00000001:1.0:1788878960.122567:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304571402 00000100:00000001:1.0:1788878960.122570:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304571945 00000100:00000001:1.0:1788878960.122572:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878960.122575:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.122576:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878960.122581:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878960/real 1788878960] req@ffff94e50449f480 x1875765290115712/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878960.122590:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878960.122591:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878960.122593:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878960.122595:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878960.122597:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122599:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878960.122601:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878960.122605:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290115712/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878960.122620:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122623:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878960.122625:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122627:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878960.122629:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122630:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878960.122642:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878960.122645:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878960.122648:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878960.122651:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878960.122654:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290115712:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878960.122658:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122661:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878960.122662:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878960.122664:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878960.122667:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878960.122668:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878960.122670:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878960.122673:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878960.122674:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878960.122677:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878960.122678:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122680:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878960.122682:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878960.122683:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878960.122685:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878960.122686:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878960.122687:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878960.122689:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878960.122690:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878960.122692:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878960.122694:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122696:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122700:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878960.122702:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.122704:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878960.122706:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878960.122708:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878960.122710:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878960.122711:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878960.122714:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878960.122715:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878960.122717:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878960.122721:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290115840:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878960.122724:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878960.122726:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878960.122728:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878960.122730:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878960.122733:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.122738:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878960.122740:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:1.0:1788878960.122744:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878960.122763:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878960.122770:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878960.122774:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878960.122777:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.122779:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878960.122785:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.122791:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304571402 00000100:00000001:1.0:1788878960.122795:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304571945 00000100:00000001:1.0:1788878960.122799:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304571402 00000100:00000001:1.0:1788878960.122803:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304571945 00000100:00000001:1.0:1788878960.122807:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878960.122811:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.122815:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878960.122823:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878960/real 1788878960] req@ffff94e50449f800 x1875765290115840/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878960.122850:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878960.122852:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878960.122856:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878960.122858:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878960.122860:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122862:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878960.122864:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878960.122869:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290115840/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878960.122877:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122880:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878960.122882:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122884:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878960.122886:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122888:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878960.122899:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878960.122902:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878960.122905:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878960.122908:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878960.122911:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290115840:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878960.122916:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122946:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878960.122948:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878960.122950:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878960.122953:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878960.122955:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878960.122956:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878960.122959:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878960.122960:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878960.122963:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878960.122964:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122967:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.122971:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878960.122973:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.122975:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878960.122977:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878960.185074:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878960.185085:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878960.185089:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878960.185099:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878960.185105:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878960.185111:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.185116:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878960.185121:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878960.185126:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878960.889027:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878960.889034:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878960.889041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878960.889043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878960.889068:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878960.889073:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878960.889078:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878961.209001:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878961.209010:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878961.209013:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878961.209019:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878961.209024:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878961.209028:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878961.209031:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878961.209035:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878961.209038:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878961.913012:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878961.913021:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878961.913027:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878961.913029:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878961.913126:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878961.913129:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878961.913132:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878962.233012:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878962.233018:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878962.233020:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878962.233024:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878962.233027:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878962.233029:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878962.233032:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878962.233034:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878962.233047:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878962.937017:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878962.937024:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878962.937029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878962.937033:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878962.937035:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878962.937039:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878962.937042:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878962.937045:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878962.937047:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878962.937049:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878962.937188:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878962.937193:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878962.937198:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878963.256984:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878963.256993:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878963.256996:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878963.257003:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878963.257007:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878963.257012:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878963.257015:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878963.257018:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878963.257021:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878963.960995:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878963.961001:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878963.961005:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878963.961006:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878963.961021:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878963.961023:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878963.961025:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878964.281012:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878964.281019:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878964.281020:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878964.281025:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878964.281028:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878964.281042:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878964.281044:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878964.281046:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878964.281049:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878964.985052:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878964.985060:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878964.985067:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878964.985068:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878964.985084:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878964.985086:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878964.985089:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878965.240974:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878965.241003:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878965.241006:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878965.241012:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878965.241029:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878965.241037:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9904 00000100:00080000:3.0:1788878965.241051:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878965.241055:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878965.241076:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878965.241089:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878965.241093:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878965.241098:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878965.241103:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878965.241108:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878965.241111:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878965.241119:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878965.241124:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878965.241166:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878965.241169:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878965.241172:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878965.241175:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878965.241178:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878965.241188:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878965.241190:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878965.241194:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878965.241196:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878965.241200:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878965.241209:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878965.241217:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290115968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878965.241225:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878965.241228:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878965.241267:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878965.241271:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878965.241275:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878965.241277:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878965.241279:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878965.241282:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878965.241285:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9904 00000100:00080000:3.0:1788878965.241290:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878965.241293:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878965.241294:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878965.241299:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878965.241302:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878965.241305:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878965.241308:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878965.241311:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878965.241313:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878965.241316:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878965.241320:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878965.241320:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878965.241327:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878965.241329:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878965.241333:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788878965.241334:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878965.241337:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788878965.241342:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878965.241345:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878965.241349:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878965.241355:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788878965.241356:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878965.241365:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878965.241367:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878965.241370:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788878965.241373:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878965.241375:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878965.241380:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878965.241381:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878965.241387:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878965.241391:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878965.241394:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290115968:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788878965.241399:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.241402:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878965.241409:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1788878965.241411:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290116096/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788878965.241413:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878965.241422:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878965.241430:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878965.241444:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878965.241447:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878965.241452:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.241454:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788878965.241455:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788878965.241460:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00080000:3.0:1788878965.241464:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788878965.241468:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878965.241468:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878965.241473:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878965.241477:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878965.241480:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9904 00000100:00080000:3.0:1788878965.241485:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878965.241487:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878965.241489:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878965.241492:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878965.241494:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878965.241496:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878965.241499:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788878965.241501:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878965.241503:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878965.241506:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878965.241509:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878965.241514:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:1.0:1788878965.241516:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788878965.241519:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878965.241523:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878965.241526:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878965.241529:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878965.241533:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00000001:3.0:1788878965.241533:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878965.241540:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.241545:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788878965.241547:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878965.241551:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.241553:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878965.241554:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878965.241558:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788878965.241563:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788878965.241581:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878965.241590:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.241592:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878965.241597:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290116224/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878965.241601:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304577065 00000100:00000001:3.0:1788878965.241612:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878965.241614:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878965.241617:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.241619:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788878965.241620:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878965.241628:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788878965.241630:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878965.241631:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878965.241634:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878965.241636:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9904 00000100:00080000:3.0:1788878965.241639:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878965.241641:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878965.241642:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878965.241645:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788878965.241646:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304577065 00000020:00000001:3.0:1788878965.241647:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878965.241650:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878965.241654:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788878965.241655:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788878965.241659:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878965.241662:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878965.241662:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878965.241669:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878965.241672:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878965.241674:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878965.241700:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878965/real 1788878965] req@ffff94e50449d180 x1875765290115968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878965.241714:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878965.241724:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878965.241732:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:1.0:1788878965.241735:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878965.241742:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788878965.241743:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.241763:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878965.241764:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788878965.241767:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878965.241769:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1788878965.241769:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878965.241774:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:3.0:1788878965.241776:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.241777:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878965.241782:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878965.241786:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290115968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788878965.241788:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878965.241791:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878965.241795:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878965.241798:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788878965.241799:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878965.241800:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.241803:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788878965.241806:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.241807:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.241811:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878965.241812:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.241816:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00100000:3.0:1788878965.241817:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290116352/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878965.241819:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878965.241825:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788878965.241831:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00080000:1.0:1788878965.241833:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788878965.241834:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.241836:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788878965.241838:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878965.241843:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290115968:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878965.241849:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.241853:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878965.241855:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878965.241858:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878965.241862:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878965.241864:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878965.241866:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878965.241870:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878965.241871:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878965.241874:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878965.241876:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.241878:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.241885:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878965.241887:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.241888:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878965.241890:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878965.241891:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878965.241892:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878965.241893:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878965.241912:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878965.241914:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878965.241916:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878965.241920:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290116096:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878965.241923:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878965.241926:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878965.241927:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878965.241929:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878965.241961:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.241969:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878965.241971:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:1.0:1788878965.241975:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878965.241992:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878965.241998:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878965.242001:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878965.242004:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.242006:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878965.242010:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.242015:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304576524 00000100:00000001:1.0:1788878965.242020:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304577065 00000100:00000001:1.0:1788878965.242024:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304576524 00000100:00000001:1.0:1788878965.242027:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304577065 00000100:00000001:1.0:1788878965.242030:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878965.242033:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.242036:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878965.242043:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878965/real 1788878965] req@ffff94e50449fb80 x1875765290116096/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878965.242057:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878965.242059:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878965.242063:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878965.242065:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878965.242067:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242069:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878965.242070:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878965.242074:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290116096/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878965.242082:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242084:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878965.242086:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242088:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878965.242089:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242090:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878965.242112:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878965.242116:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878965.242121:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878965.242124:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878965.242145:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290116096:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878965.242152:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878965.242153:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878965.242172:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878965.242174:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878965.242176:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878965.242178:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878965.242182:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290116224:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878965.242185:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878965.242188:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878965.242189:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878965.242191:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878965.242194:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.242200:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878965.242201:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:1.0:1788878965.242204:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878965.242214:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878965.242217:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878965.242219:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878965.242220:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.242238:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878965.242242:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.242246:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304576522 00000100:00000001:1.0:1788878965.242249:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304577065 00000100:00000001:1.0:1788878965.242251:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304576522 00000100:00000001:1.0:1788878965.242254:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304577065 00000100:00000001:1.0:1788878965.242256:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878965.242258:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.242261:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878965.242265:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878965/real 1788878965] req@ffff94e50449d880 x1875765290116224/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878965.242274:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878965.242275:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878965.242277:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878965.242279:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878965.242281:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242283:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878965.242285:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878965.242289:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290116224/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878965.242297:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242299:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878965.242302:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242303:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878965.242305:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242307:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878965.242317:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878965.242320:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878965.242323:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878965.242325:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878965.242329:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290116224:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878965.242332:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878965.242334:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878965.242335:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878965.242338:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878965.242340:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878965.242342:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878965.242345:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290116352:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878965.242348:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878965.242350:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878965.242352:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878965.242354:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878965.242356:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.242361:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878965.242363:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:1.0:1788878965.242366:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878965.242378:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878965.242382:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878965.242384:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878965.242386:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.242387:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878965.242390:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.242393:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304576522 00000100:00000001:1.0:1788878965.242396:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304577065 00000100:00000001:1.0:1788878965.242398:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304576522 00000100:00000001:1.0:1788878965.242401:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304577065 00000100:00000001:1.0:1788878965.242403:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878965.242405:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.242407:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878965.242410:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878965/real 1788878965] req@ffff94e50449c380 x1875765290116352/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878965.242418:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878965.242419:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878965.242421:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878965.242423:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878965.242424:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242426:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878965.242427:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878965.242431:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290116352/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878965.242438:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242440:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878965.242443:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242445:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878965.242446:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242448:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878965.242459:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878965.242461:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878965.242465:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878965.242467:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878965.242470:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290116352:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878965.242474:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242476:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878965.242478:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878965.242479:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878965.242482:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878965.242483:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878965.242485:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878965.242487:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878965.242488:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878965.242491:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878965.242492:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242494:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878965.242495:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878965.242497:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878965.242498:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878965.242499:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878965.242501:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878965.242502:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878965.242503:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878965.242505:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878965.242506:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242508:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878965.242509:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878965.242510:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878965.242512:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878965.242513:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878965.242514:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878965.242515:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878965.242516:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878965.242518:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878965.242519:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242521:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.242525:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878965.242526:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.242528:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878965.242530:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878965.304985:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878965.304994:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878965.304998:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878965.305006:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878965.305012:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878965.305018:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.305023:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878965.305028:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878965.305032:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878966.009036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878966.009046:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878966.009049:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878966.009055:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878966.009063:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878966.009067:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878966.009071:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878966.009076:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878966.009082:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878966.009085:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878966.009218:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878966.009224:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878966.009229:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878966.329061:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878966.329072:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878966.329075:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878966.329085:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878966.329090:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878966.329095:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878966.329099:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878966.329102:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878966.329105:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878967.033043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878967.033052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878967.033060:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878967.033062:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878967.033143:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878967.033151:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878967.033158:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878967.352970:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878967.352976:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878967.352978:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878967.352982:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878967.352985:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878967.352987:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878967.352989:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878967.352991:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878967.352993:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878968.057077:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878968.057086:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878968.057092:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878968.057093:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878968.057134:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878968.057137:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878968.057140:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878968.377014:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878968.377020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878968.377021:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878968.377025:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878968.377028:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878968.377031:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878968.377034:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878968.377036:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878968.377039:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878969.081009:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878969.081014:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878969.081023:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878969.081024:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878969.081062:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878969.081066:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878969.081074:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878969.401024:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878969.401032:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878969.401035:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878969.401041:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878969.401046:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878969.401052:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878969.401057:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878969.401061:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878969.401066:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878970.104999:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878970.105007:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878970.105013:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878970.105015:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878970.105054:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878970.105058:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878970.105061:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878970.361037:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878970.361047:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878970.361050:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878970.361054:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878970.361058:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878970.361066:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9909 00000100:00080000:3.0:1788878970.361077:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878970.361080:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878970.361082:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878970.361110:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878970.361113:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878970.361116:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878970.361119:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878970.361121:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878970.361124:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878970.361128:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878970.361131:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878970.361134:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878970.361138:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878970.361140:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878970.361143:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878970.361145:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878970.361157:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878970.361158:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878970.361161:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878970.361164:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878970.361179:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878970.361192:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878970.361201:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290116480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878970.361212:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878970.361216:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878970.361249:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878970.361252:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878970.361255:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878970.361257:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878970.361259:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878970.361261:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878970.361264:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9909 00000100:00080000:3.0:1788878970.361268:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878970.361270:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878970.361272:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878970.361276:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878970.361277:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878970.361279:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878970.361281:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788878970.361301:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788878970.361302:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878970.361306:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878970.361309:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788878970.361313:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878970.361315:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788878970.361319:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.361321:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788878970.361325:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878970.361326:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878970.361334:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878970.361335:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788878970.361341:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878970.361343:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878970.361344:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878970.361351:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788878970.361352:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788878970.361356:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878970.361360:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878970.361366:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1788878970.361368:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290116480:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788878970.361368:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878970.361373:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878970.361376:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878970.361380:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.361383:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788878970.361388:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878970.361407:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878970.361414:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878970.361415:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290116608/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788878970.361423:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788878970.361429:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878970.361431:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878970.361434:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878970.361444:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788878970.361446:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878970.361447:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878970.361452:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878970.361453:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878970.361455:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878970.361476:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9909 00000100:00080000:3.0:1788878970.361480:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878970.361482:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878970.361484:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788878970.361485:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878970.361491:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000020:00000001:3.0:1788878970.361492:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878970.361495:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878970.361498:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788878970.361502:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788878970.361504:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788878970.361505:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878970.361510:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878970.361514:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878970.361516:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878970.361518:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878970.361521:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878970.361523:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878970.361525:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878970.361528:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878970.361530:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878970.361532:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878970.361536:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878970.361537:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878970.361539:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878970.361541:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878970.361543:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878970.361550:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878970.361554:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290116736/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788878970.361562:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788878970.361570:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878970.361573:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878970.361576:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.361596:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:3.0:1788878970.361597:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878970.361602:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788878970.361604:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878970.361605:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788878970.361610:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878970.361611:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878970.361617:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878970.361619:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878970.361624:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9909 00000100:00000001:1.0:1788878970.361625:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878970.361630:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878970.361632:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304582185 00000100:00000001:3.0:1788878970.361633:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878970.361637:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878970.361641:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878970.361642:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878970.361644:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788878970.361662:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1788878970.361663:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878970.361669:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878970.361671:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878970.361674:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304582185 00000100:00080000:3.0:1788878970.361675:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878970.361684:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.361685:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1788878970.361691:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878970.361694:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878970.361696:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878970.361699:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878970.361701:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878970.361712:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878970.361716:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878970.361718:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000008:00000001:3.0:1788878970.361720:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878970.361724:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878970.361731:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878970/real 1788878970] req@ffff94e50449ce00 x1875765290116480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788878970.361732:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878970.361735:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878970.361740:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878970.361742:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878970.361745:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878970.361753:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878970.361764:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290116864/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1788878970.361768:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878970.361774:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788878970.361775:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878970.361779:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.361784:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788878970.361784:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.361801:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878970.361807:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.361810:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878970.361812:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878970.361820:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290116480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788878995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878970.361833:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.361838:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878970.361842:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.361845:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878970.361847:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.361850:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878970.361853:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878970.361884:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878970.361890:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878970.361894:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878970.361900:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290116480:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878970.361907:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.361910:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878970.361912:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878970.361915:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878970.361918:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878970.361920:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878970.361922:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878970.361926:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878970.361927:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878970.361941:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878970.361942:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.361945:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.361951:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878970.361953:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.361955:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878970.361956:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878970.361958:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878970.361959:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878970.361960:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878970.361963:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878970.361965:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878970.361967:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878970.361970:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290116608:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878970.361973:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878970.361975:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878970.361977:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878970.361979:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878970.361982:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.361989:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878970.361990:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:1.0:1788878970.362029:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878970.362046:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878970.362052:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878970.362055:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878970.362057:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.362059:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878970.362063:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.362068:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304581644 00000100:00000001:1.0:1788878970.362072:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304582185 00000100:00000001:1.0:1788878970.362076:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304581644 00000100:00000001:1.0:1788878970.362079:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304582185 00000100:00000001:1.0:1788878970.362083:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878970.362085:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.362088:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878970.362111:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878970/real 1788878970] req@ffff94e50449df80 x1875765290116608/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878970.362124:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878970.362126:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878970.362129:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878970.362131:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878970.362132:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362134:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878970.362135:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878970.362139:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290116608/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788878995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878970.362146:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362148:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878970.362150:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362151:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878970.362153:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362154:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878970.362165:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878970.362167:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878970.362169:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878970.362171:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878970.362174:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290116608:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878970.362184:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878970.362185:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878970.362187:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878970.362189:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878970.362191:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878970.362192:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878970.362195:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290116736:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878970.362198:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878970.362200:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878970.362202:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878970.362203:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878970.362206:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.362211:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878970.362212:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000100:00000001:1.0:1788878970.362216:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878970.362228:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878970.362232:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878970.362234:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878970.362236:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.362247:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878970.362250:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.362253:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304581642 00000100:00000001:1.0:1788878970.362255:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304582185 00000100:00000001:1.0:1788878970.362257:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304581642 00000100:00000001:1.0:1788878970.362259:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304582185 00000100:00000001:1.0:1788878970.362261:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878970.362262:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.362264:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878970.362285:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878970/real 1788878970] req@ffff94e50449e680 x1875765290116736/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878970.362293:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878970.362294:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878970.362296:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878970.362298:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878970.362299:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362301:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878970.362303:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878970.362307:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290116736/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878970.362324:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362326:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878970.362329:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362347:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878970.362349:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362351:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878970.362366:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878970.362371:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878970.362376:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878970.362379:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878970.362383:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290116736:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878970.362389:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878970.362391:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878970.362393:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878970.362413:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878970.362415:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878970.362418:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878970.362423:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290116864:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878970.362427:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878970.362430:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878970.362432:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878970.362434:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878970.362449:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.362456:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878970.362457:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:1.0:1788878970.362461:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878970.362493:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878970.362500:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878970.362503:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878970.362505:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.362508:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878970.362512:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.362517:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304581642 00000100:00000001:1.0:1788878970.362521:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304582185 00000100:00000001:1.0:1788878970.362525:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304581642 00000100:00000001:1.0:1788878970.362528:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304582185 00000100:00000001:1.0:1788878970.362531:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878970.362534:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.362537:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878970.362545:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878970/real 1788878970] req@ffff94e50449c380 x1875765290116864/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878970.362558:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878970.362560:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878970.362563:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878970.362565:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878970.362567:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362569:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878970.362570:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878970.362574:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290116864/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788878995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878970.362581:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362600:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878970.362602:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362603:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878970.362604:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362606:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878970.362615:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878970.362617:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878970.362620:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878970.362621:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878970.362625:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290116864:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878970.362628:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362630:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878970.362631:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878970.362633:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878970.362634:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878970.362636:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878970.362637:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878970.362639:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878970.362640:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878970.362642:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878970.362644:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362645:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878970.362646:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878970.362647:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878970.362649:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878970.362650:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878970.362651:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878970.362652:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878970.362653:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878970.362654:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878970.362655:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362656:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878970.362678:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878970.362679:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878970.362681:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878970.362682:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878970.362682:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878970.362684:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878970.362685:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878970.362686:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878970.362687:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362689:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.362693:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878970.362694:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.362696:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878970.362698:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878970.425013:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878970.425021:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878970.425023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878970.425030:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878970.425047:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878970.425051:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.425054:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878970.425057:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878970.425060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878971.128993:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878971.129012:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878971.129020:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878971.129022:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878971.129063:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878971.129067:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878971.129069:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878971.449117:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878971.449145:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878971.449148:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878971.449157:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878971.449163:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878971.449168:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878971.449172:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878971.449176:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878971.449180:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878972.153020:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878972.153027:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878972.153032:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878972.153036:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878972.153038:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878972.153065:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878972.153068:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878972.153070:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878972.153072:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878972.153074:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878972.153126:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878972.153128:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878972.153131:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878972.473004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878972.473013:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878972.473016:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878972.473021:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878972.473025:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878972.473046:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878972.473048:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878972.473051:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878972.473054:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878973.177069:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878973.177111:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878973.177118:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878973.177121:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878973.177151:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878973.177155:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878973.177159:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878973.497060:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878973.497102:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878973.497107:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878973.497115:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878973.497122:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878973.497128:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878973.497133:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878973.497137:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878973.497142:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878974.201094:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878974.201103:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878974.201110:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878974.201111:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878974.201186:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878974.201191:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878974.201195:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878974.521052:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878974.521081:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878974.521086:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878974.521095:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878974.521100:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878974.521105:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878974.521136:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878974.521139:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878974.521143:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878975.225029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878975.225041:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878975.225045:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878975.225050:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878975.225079:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878975.225084:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878975.225089:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878975.225093:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878975.225101:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878975.225103:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878975.225193:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878975.225200:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878975.225206:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788878975.481106:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878975.481121:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788878975.481125:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878975.481131:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878975.481137:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878975.481147:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9914 00000100:00080000:0.0:1788878975.481162:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878975.481166:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878975.481168:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878975.481179:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878975.481183:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878975.481187:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788878975.481190:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788878975.481193:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878975.481194:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788878975.481200:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878975.481204:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878975.481208:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878975.481213:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878975.481217:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878975.481220:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878975.481223:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878975.481240:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878975.481242:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878975.481246:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878975.481249:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878975.481256:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878975.481272:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878975.481283:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290116992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878975.481296:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878975.481303:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878975.481326:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878975.481339:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878975.481344:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788878975.481347:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878975.481349:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878975.481354:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878975.481358:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9914 00000100:00080000:0.0:1788878975.481363:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878975.481365:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878975.481367:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878975.481372:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788878975.481373:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788878975.481375:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788878975.481386:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788878975.481387:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788878975.481395:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788878975.481395:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878975.481402:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878975.481415:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878975.481418:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878975.481422:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878975.481424:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878975.481431:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878975.481432:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878975.481435:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788878975.481439:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878975.481444:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878975.481445:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878975.481463:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878975.481471:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:1.0:1788878975.481473:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290116992:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1788878975.481479:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788878975.481480:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1788878975.481489:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878975.481492:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788878975.481494:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878975.481497:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878975.481500:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878975.481501:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878975.481513:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878975.481516:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878975.481518:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788878975.481520:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878975.481521:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878975.481549:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.481552:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878975.481556:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00100000:0.0:1788878975.481563:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290117120/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878975.481565:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788878975.481578:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878975.481581:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878975.481587:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878975.481590:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788878975.481595:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788878975.481615:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788878975.481618:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788878975.481624:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878975.481628:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788878975.481634:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9914 00000100:00000001:1.0:1788878975.481641:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:0.0:1788878975.481642:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878975.481648:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878975.481650:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788878975.481654:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788878975.481657:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788878975.481659:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1788878975.481663:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788878975.481664:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.481670:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:0.0:1788878975.481683:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788878975.481686:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.481697:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304587305 00000100:00000001:0.0:1788878975.481698:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878975.481706:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878975.481711:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1788878975.481720:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878975.481724:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304587305 00000100:00000001:0.0:1788878975.481731:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.481733:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788878975.481743:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878975.481744:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788878975.481747:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.481753:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1788878975.481755:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788878975.481764:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878975/real 1788878975] req@ffff94e508978e00 x1875765290116992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788878975.481766:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878975.481778:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878975.481790:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000400:00000001:1.0:1788878975.481791:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878975.481797:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878975.481817:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878975.481820:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878975.481821:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.481824:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878975.481826:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788878975.481830:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878975.481832:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290116992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878975.481841:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.481860:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788878975.481861:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878975.481865:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.481866:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878975.481878:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788878975.481878:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788878975.481886:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788878975.481888:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1788878975.481913:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.481965:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.481969:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878975.481970:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878975.481974:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878975.481978:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878975.481980:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878975.481983:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290116992:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878975.481987:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.481990:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878975.481992:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1788878975.481997:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878975.481998:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878975.482009:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878975.482017:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878975.482030:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00100000:0.0:1788878975.482032:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290117248/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788878975.482034:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878975.482040:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878975.482053:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878975.482053:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878975.482071:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482073:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482079:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878975.482082:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482083:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878975.482090:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482092:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1788878975.482102:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788878975.482103:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878975.482113:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1788878975.482119:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788878975.482121:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878975.482123:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788878975.482125:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878975.482128:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1788878975.482130:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878975.482137:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878975.482138:0:178134:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:1.0:1788878975.482146:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788878975.482149:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9914 00000100:00100000:1.0:1788878975.482155:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290117120:192.168.203.129@tcp:38:kworker.0 00000100:00080000:0.0:1788878975.482157:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878975.482168:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788878975.482169:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788878975.482172:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788878975.482173:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788878975.482178:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878975.482182:0:178134:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1788878975.482182:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1788878975.482189:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1788878975.482191:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878975.482194:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788878975.482199:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878975.482202:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878975.482202:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878975.482213:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788878975.482213:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:1.0:1788878975.482228:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788878975.482229:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878975.482237:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878975.482241:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788878975.482246:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878975.482249:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878975.482251:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788878975.482260:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1788878975.482261:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00000001:0.0:1788878975.482268:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482270:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878975.482272:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482277:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1788878975.482279:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788878975.482280:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878975.482285:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482286:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878975.482293:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788878975.482295:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878975.482296:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878975.482304:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482306:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304586764 00000100:00000001:1.0:1788878975.482323:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304587305 00000100:00000001:1.0:1788878975.482327:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304586764 00000100:00000001:0.0:1788878975.482330:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482347:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304587305 00000100:00100000:0.0:1788878975.482352:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290117376/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878975.482355:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878975.482369:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482384:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878975.482399:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878975/real 1788878975] req@ffff94e50897bb80 x1875765290117120/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878975.482400:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878975.482412:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878975.482424:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788878975.482432:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878975.482434:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878975.482443:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878975.482447:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878975.482448:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482450:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878975.482452:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878975.482458:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290117120/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878975.482467:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482469:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878975.482472:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482473:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878975.482474:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482475:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878975.482486:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878975.482489:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878975.482492:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878975.482494:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878975.482497:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290117120:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788878975.482501:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878975.482502:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878975.482503:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878975.482506:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878975.482508:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878975.482510:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878975.482513:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290117248:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878975.482515:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878975.482518:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878975.482519:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878975.482521:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878975.482524:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482529:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878975.482530:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:1.0:1788878975.482534:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878975.482544:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878975.482547:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878975.482549:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878975.482550:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482552:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878975.482554:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482557:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304586762 00000100:00000001:1.0:1788878975.482559:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304587305 00000100:00000001:1.0:1788878975.482561:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304586762 00000100:00000001:1.0:1788878975.482563:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304587305 00000100:00000001:1.0:1788878975.482565:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878975.482566:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482568:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878975.482571:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878975/real 1788878975] req@ffff94e50897ad80 x1875765290117248/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878975.482577:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878975.482578:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878975.482580:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878975.482581:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878975.482582:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482584:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878975.482585:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878975.482588:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290117248/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878975.482600:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482601:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878975.482604:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482605:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878975.482606:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482607:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878975.482617:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878975.482619:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878975.482621:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878975.482623:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878975.482625:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290117248:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878975.482628:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482630:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878975.482632:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878975.482633:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878975.482635:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878975.482636:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878975.482637:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878975.482639:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878975.482641:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878975.482643:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878975.482644:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482646:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878975.482647:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878975.482648:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878975.482649:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878975.482650:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878975.482651:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878975.482652:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878975.482653:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878975.482655:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878975.482656:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482657:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482660:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878975.482662:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482664:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878975.482666:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878975.482667:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878975.482669:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878975.482670:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878975.482672:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878975.482673:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878975.482675:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878975.482677:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290117376:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878975.482680:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788878975.482682:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788878975.482683:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878975.482685:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788878975.482687:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482690:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878975.482692:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:1.0:1788878975.482694:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788878975.482702:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878975.482706:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788878975.482707:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878975.482708:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482710:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878975.482712:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482715:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304586762 00000100:00000001:1.0:1788878975.482717:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304587305 00000100:00000001:1.0:1788878975.482719:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304586762 00000100:00000001:1.0:1788878975.482721:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304587305 00000100:00000001:1.0:1788878975.482723:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878975.482724:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482726:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788878975.482729:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878975/real 1788878975] req@ffff94e508979c00 x1875765290117376/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788878975.482740:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788878975.482742:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788878975.482744:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878975.482745:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878975.482746:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482747:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878975.482748:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878975.482751:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290117376/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878975.482756:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482758:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788878975.482760:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482761:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788878975.482762:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482763:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788878975.482784:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788878975.482786:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788878975.482789:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878975.482790:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788878975.482793:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290117376:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788878975.482796:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482797:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878975.482798:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788878975.482800:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788878975.482802:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788878975.482803:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788878975.482803:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788878975.482805:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788878975.482806:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788878975.482808:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788878975.482809:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482811:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.482813:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878975.482814:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.482816:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788878975.482817:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878975.545107:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878975.545118:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878975.545123:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878975.545132:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878975.545137:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878975.545142:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.545146:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878975.545149:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878975.545152:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878976.248989:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878976.248994:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878976.249000:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878976.249002:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878976.249024:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878976.249028:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878976.249031:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878976.568972:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878976.568977:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878976.568979:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878976.568983:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878976.568986:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878976.568989:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878976.568991:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878976.568993:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878976.568995:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878977.273016:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878977.273024:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878977.273031:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878977.273033:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878977.273069:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878977.273072:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878977.273074:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878977.592988:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878977.592996:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878977.592998:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878977.593019:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878977.593022:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878977.593025:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878977.593028:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878977.593029:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878977.593031:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878978.297025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878978.297033:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878978.297064:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878978.297066:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878978.297191:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878978.297196:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878978.297202:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878978.616962:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878978.616970:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878978.616972:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878978.616977:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878978.616981:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878978.616985:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878978.616989:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878978.616991:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878978.616994:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878979.320985:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878979.320991:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878979.320996:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878979.320997:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878979.321082:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878979.321087:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878979.321090:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878979.641004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878979.641010:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878979.641013:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878979.641017:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878979.641021:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878979.641025:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878979.641043:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878979.641045:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878979.641048:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878980.345056:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878980.345074:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878980.345090:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878980.345094:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878980.345129:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878980.345134:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878980.345140:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788878980.601031:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878980.601043:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788878980.601046:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878980.601049:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878980.601053:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878980.601061:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9919 00000100:00080000:3.0:1788878980.601071:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878980.601074:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878980.601076:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878980.601083:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878980.601086:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878980.601089:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788878980.601091:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788878980.601093:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878980.601095:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878980.601098:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878980.601101:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878980.601104:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788878980.601106:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878980.601109:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878980.601112:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788878980.601114:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878980.601123:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878980.601124:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878980.601127:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878980.601129:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878980.601133:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878980.601143:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788878980.601151:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290117504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788878980.601165:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878980.601170:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878980.601195:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878980.601198:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788878980.601201:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788878980.601203:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878980.601205:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878980.601208:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878980.601210:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9919 00000100:00080000:3.0:1788878980.601214:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878980.601216:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878980.601218:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788878980.601221:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878980.601222:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878980.601224:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788878980.601225:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788878980.601229:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788878980.601230:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878980.601232:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878980.601235:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878980.601236:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788878980.601237:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601240:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788878980.601240:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878980.601246:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788878980.601248:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878980.601249:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878980.601254:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878980.601257:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878980.601259:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878980.601265:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290117504:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878980.601269:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878980.601273:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878980.601275:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878980.601277:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878980.601282:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601308:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878980.601310:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:0.0:1788878980.601314:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788878980.601325:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878980.601327:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878980.601330:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878980.601333:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878980.601335:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878980.601339:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878980.601340:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878980.601343:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878980.601344:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878980.601346:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878980.601351:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788878980.601353:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1788878980.601359:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290117632/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878980.601364:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788878980.601368:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788878980.601371:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788878980.601372:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601374:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601378:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788878980.601379:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788878980.601383:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878980.601384:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601391:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304592425 00000100:00080000:3.0:1788878980.601391:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788878980.601393:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878980.601397:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878980.601399:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788878980.601402:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9919 00000100:00000001:0.0:1788878980.601403:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788878980.601405:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788878980.601409:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878980.601411:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878980.601412:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304592425 00000020:00000001:3.0:1788878980.601419:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878980.601421:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:3.0:1788878980.601422:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788878980.601426:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878980.601428:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788878980.601431:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878980.601432:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878980.601437:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878980.601439:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788878980.601440:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878980.601445:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788878980.601447:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878980.601450:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:0.0:1788878980.601452:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878980/real 1788878980] req@ffff94e50449d880 x1875765290117504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788878980.601453:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788878980.601458:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878980.601462:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788878980.601465:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788878980.601470:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878980.601474:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878980.601480:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878980.601483:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788878980.601484:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878980.601488:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878980.601492:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878980.601495:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878980.601497:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.601500:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878980.601502:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878980.601509:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290117504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878980.601525:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.601528:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878980.601531:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.601532:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878980.601533:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.601535:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878980.601537:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878980.601539:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878980.601543:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878980.601545:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878980.601548:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290117504:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878980.601552:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.601554:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878980.601556:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878980.601558:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878980.601562:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878980.601563:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878980.601565:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878980.601568:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878980.601570:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878980.601573:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878980.601574:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.601576:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.601581:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878980.601583:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601585:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878980.601586:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878980.601588:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878980.601589:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878980.601590:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878980.601592:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878980.601594:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788878980.601598:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788878980.601599:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878980.601602:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1788878980.601605:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290117632:192.168.203.129@tcp:38:kworker.0 02000000:00000001:3.0:1788878980.601606:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601613:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878980.601616:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788878980.601617:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878980.601618:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878980.601622:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878980.601625:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601632:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878980.601634:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:0.0:1788878980.601638:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878980.601652:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878980.601658:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00100000:3.0:1788878980.601659:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290117760/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878980.601662:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878980.601666:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601670:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878980.601675:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878980.601675:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788878980.601681:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601682:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304591883 00000100:00000001:0.0:1788878980.601688:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304592425 00000100:00000001:0.0:1788878980.601691:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304591883 00000100:00000001:3.0:1788878980.601691:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601697:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304592425 00000100:00000001:0.0:1788878980.601700:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788878980.601700:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878980.601705:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601708:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788878980.601713:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:0.0:1788878980.601716:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878980/real 1788878980] req@ffff94e50449fb80 x1875765290117632/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788878980.601718:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788878980.601724:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788878980.601735:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000400:00000001:0.0:1788878980.601738:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878980.601743:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1788878980.601744:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9919 00000100:00000001:0.0:1788878980.601750:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:3.0:1788878980.601753:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878980.601756:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788878980.601759:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878980.601760:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878980.601764:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878980.601766:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878980.601771:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788878980.601773:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788878980.601777:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:0.0:1788878980.601781:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290117632/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788878980.601781:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788878980.601786:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788878980.601792:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788878980.601794:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788878980.601803:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878980.601808:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788878980.601808:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878980.601815:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878980.601816:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788878980.601818:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878980.601823:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788878980.601823:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878980.601829:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878980.601831:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00100000:3.0:1788878980.601832:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878980.601835:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1788878980.601836:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601842:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788878980.601842:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788878980.601847:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788878980.601850:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878980.601852:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788878980.601853:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788878980.601856:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878980.601861:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601862:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:3.0:1788878980.601868:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290117888/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1788878980.601869:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878980.601878:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878980.601884:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788878980.601889:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1788878980.601891:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290117632:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788878980.601893:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788878980.601897:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601900:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.601934:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878980.601936:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878980.601938:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878980.601941:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878980.601943:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878980.601945:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878980.601947:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878980.601948:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878980.601951:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878980.601953:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.601954:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.601958:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878980.601960:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601963:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878980.601964:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878980.601966:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878980.601967:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878980.601968:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878980.601970:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878980.601972:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878980.601974:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878980.601978:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290117760:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878980.601981:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878980.601983:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878980.601984:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878980.601986:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878980.601989:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.601995:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878980.601996:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:0.0:1788878980.602000:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878980.602011:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878980.602015:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878980.602017:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878980.602018:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.602037:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878980.602041:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.602044:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304591882 00000100:00000001:0.0:1788878980.602046:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304592425 00000100:00000001:0.0:1788878980.602048:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304591882 00000100:00000001:0.0:1788878980.602050:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304592425 00000100:00000001:0.0:1788878980.602051:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878980.602053:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.602054:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878980.602058:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878980/real 1788878980] req@ffff94e50449d180 x1875765290117760/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878980.602064:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878980.602065:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878980.602067:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878980.602068:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878980.602069:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.602071:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878980.602072:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878980.602075:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290117760/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878980.602080:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.602082:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878980.602084:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.602085:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878980.602086:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.602087:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878980.602096:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878980.602099:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878980.602101:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878980.602103:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878980.602105:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290117760:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878980.602108:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878980.602109:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878980.602110:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878980.602111:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878980.602112:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878980.602114:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878980.602116:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290117888:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878980.602119:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878980.602120:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878980.602121:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878980.602122:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878980.602125:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.602128:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878980.602129:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:0.0:1788878980.602131:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878980.602140:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878980.602143:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878980.602145:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878980.602146:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.602147:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878980.602149:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.602151:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304591882 00000100:00000001:0.0:1788878980.602153:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304592425 00000100:00000001:0.0:1788878980.602154:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304591882 00000100:00000001:0.0:1788878980.602156:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304592425 00000100:00000001:0.0:1788878980.602157:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878980.602159:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.602161:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878980.602163:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878980/real 1788878980] req@ffff94e50449f800 x1875765290117888/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878980.602169:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878980.602170:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878980.602171:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878980.602172:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878980.602173:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.602175:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878980.602176:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878980.602178:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290117888/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878980.602183:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.602184:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878980.602186:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.602187:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878980.602187:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.602189:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878980.602197:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878980.602199:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878980.602201:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878980.602202:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878980.602204:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290117888:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878980.602207:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.602208:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878980.602209:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878980.602210:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878980.602211:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878980.602212:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878980.602213:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878980.602215:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878980.602216:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878980.602218:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878980.602219:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.602220:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878980.602221:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878980.602222:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878980.602223:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878980.602224:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878980.602225:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878980.602226:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878980.602226:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878980.602227:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878980.602228:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.602229:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878980.602232:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878980.602233:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878980.602234:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878980.602235:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878980.664992:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878980.664998:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878980.665000:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878980.665005:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878980.665008:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878980.665011:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878980.665013:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878980.665015:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878980.665018:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878981.368977:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878981.368983:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878981.368986:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878981.368987:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878981.369004:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878981.369006:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878981.369009:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878981.689091:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878981.689102:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878981.689105:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878981.689112:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878981.689116:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878981.689121:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878981.689125:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878981.689128:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878981.689131:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878982.393032:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878982.393039:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878982.393044:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878982.393047:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878982.393050:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878982.393053:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878982.393056:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878982.393059:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878982.393060:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878982.393062:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878982.393133:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878982.393136:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878982.393140:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878982.713166:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878982.713184:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878982.713188:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878982.713201:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878982.713209:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878982.713218:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878982.713224:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878982.713229:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878982.713235:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878983.417096:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878983.417108:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878983.417118:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878983.417121:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878983.417225:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878983.417230:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878983.417234:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878983.737135:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878983.737147:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878983.737151:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878983.737158:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878983.737164:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878983.737170:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878983.737174:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878983.737178:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878983.737181:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878984.441129:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878984.441141:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878984.441152:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878984.441154:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878984.441188:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878984.441192:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878984.441195:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878984.761021:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878984.761029:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878984.761030:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878984.761036:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878984.761039:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878984.761044:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878984.761046:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878984.761049:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878984.761051:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878985.465092:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878985.465105:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878985.465109:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878985.465113:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878985.465119:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878985.465122:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878985.465124:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878985.465127:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878985.465131:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878985.465132:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878985.465162:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878985.465166:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878985.465169:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788878985.721088:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878985.721123:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788878985.721128:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788878985.721134:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878985.721141:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788878985.721151:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9924 00000100:00080000:1.0:1788878985.721167:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878985.721173:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878985.721177:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788878985.721189:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788878985.721193:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788878985.721197:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788878985.721200:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788878985.721203:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878985.721205:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788878985.721210:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878985.721213:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878985.721217:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788878985.721221:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878985.721224:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721227:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788878985.721230:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878985.721244:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788878985.721246:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721249:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788878985.721252:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788878985.721258:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721308:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878985.721317:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100bb80 x1875765290118016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878985.721326:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878985.721331:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721369:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788878985.721373:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878985.721377:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788878985.721379:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788878985.721381:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878985.721385:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788878985.721388:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9924 00000100:00080000:1.0:1788878985.721393:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878985.721395:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878985.721397:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788878985.721401:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788878985.721403:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788878985.721405:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788878985.721408:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788878985.721410:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878985.721412:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788878985.721415:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878985.721418:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878985.721420:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788878985.721442:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878985.721444:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721445:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788878985.721449:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788878985.721452:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721454:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878985.721457:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788878985.721458:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721459:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788878985.721461:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788878985.721463:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721467:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878985.721472:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008700 x1875765290118144/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878985.721480:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878985.721483:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721486:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788878985.721489:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878985.721493:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788878985.721496:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788878985.721498:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878985.721501:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788878985.721503:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9924 00000100:00080000:1.0:1788878985.721507:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878985.721509:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878985.721510:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878985.721513:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1788878985.721513:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788878985.721515:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788878985.721521:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878985.721523:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1788878985.721526:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878985.721528:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788878985.721530:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878985.721534:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878985.721535:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878985.721543:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788878985.721547:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878985.721554:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878985.721555:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878985.721561:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788878985.721565:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878985.721568:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721569:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788878985.721571:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788878985.721573:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:1.0:1788878985.721574:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878985.721588:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721590:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878985.721609:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788878985.721612:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721615:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:0.0:1788878985.721616:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290118016:192.168.203.129@tcp:250:kworker.0 02000000:00000001:1.0:1788878985.721618:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788878985.721622:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.721626:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788878985.721630:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878985.721633:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878985.721637:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878985.721639:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008e00 x1875765290118272/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788878985.721642:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788878985.721652:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878985.721654:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721669:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878985.721672:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788878985.721673:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878985.721683:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788878985.721685:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788878985.721686:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878985.721689:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788878985.721692:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9924 00000100:00080000:1.0:1788878985.721696:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878985.721698:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878985.721699:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878985.721700:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:1.0:1788878985.721703:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788878985.721704:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286523264 : -117763423028352 : ffff94e51100bb80) 00000020:00000001:1.0:1788878985.721708:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878985.721711:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1788878985.721711:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:1.0:1788878985.721719:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788878985.721732:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878985.721733:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788878985.721737:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878985.721740:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878985.721742:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788878985.721745:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878985.721747:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721749:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788878985.721752:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788878985.721754:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721756:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878985.721759:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788878985.721760:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721761:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788878985.721763:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788878985.721765:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.721769:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878985.721772:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008000 x1875765290118400/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878985.721779:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878985.721784:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788878985.721785:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878985.721788:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.721803:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878985.721813:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878985.721816:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.721818:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878985.721823:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.721827:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304597545 00000100:00000001:0.0:1788878985.721840:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788878985.721849:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304597545 00000100:00000001:0.0:1788878985.721853:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788878985.721856:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878985.721859:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.721861:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878985.721868:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878985/real 1788878985] req@ffff94e51100bb80 x1875765290118016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878985.721894:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878985.721896:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878985.721899:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878985.721901:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878985.721902:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.721905:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878985.721906:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878985.721910:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100bb80 x1875765290118016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878985.721915:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.721918:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878985.721920:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.721921:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878985.721922:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.721924:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878985.721925:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878985.721928:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878985.721932:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878985.721934:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878985.721937:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290118016:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878985.721941:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878985.721943:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878985.721944:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878985.721982:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878985.721998:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878985.722000:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878985.722003:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290118144:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878985.722005:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878985.722008:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878985.722009:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878985.722010:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878985.722013:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.722018:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878985.722019:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286509824 : -117763423041792 : ffff94e511008700) 00000100:00000001:0.0:1788878985.722023:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878985.722032:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878985.722036:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878985.722038:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878985.722039:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.722040:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878985.722042:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.722056:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304597004 00000100:00000001:0.0:1788878985.722058:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304597545 00000100:00000001:0.0:1788878985.722060:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304597004 00000100:00000001:0.0:1788878985.722062:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304597545 00000100:00000001:0.0:1788878985.722063:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878985.722064:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.722066:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878985.722068:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878985/real 1788878985] req@ffff94e511008700 x1875765290118144/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878985.722079:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878985.722079:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878985.722081:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878985.722082:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878985.722083:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722084:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878985.722086:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878985.722088:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008700 x1875765290118144/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878985.722093:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722094:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878985.722095:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722096:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878985.722097:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722098:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878985.722114:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878985.722116:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878985.722118:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878985.722120:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878985.722122:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290118144:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878985.722125:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722127:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878985.722128:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878985.722131:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878985.722134:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878985.722137:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878985.722139:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878985.722143:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878985.722145:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878985.722148:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878985.722149:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722151:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878985.722152:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878985.722153:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878985.722154:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878985.722155:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878985.722156:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878985.722157:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878985.722157:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878985.722158:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878985.722159:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722161:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722169:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878985.722170:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.722172:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878985.722173:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878985.722174:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878985.722175:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878985.722176:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878985.722177:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878985.722179:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878985.722180:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878985.722182:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290118272:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878985.722185:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878985.722186:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878985.722187:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878985.722188:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878985.722190:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.722194:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878985.722194:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286511616 : -117763423040000 : ffff94e511008e00) 00000100:00000001:0.0:1788878985.722197:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878985.722204:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878985.722206:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878985.722207:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878985.722208:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.722210:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878985.722211:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.722214:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304597002 00000100:00000001:0.0:1788878985.722216:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304597545 00000100:00000001:0.0:1788878985.722218:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304597002 00000100:00000001:0.0:1788878985.722219:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304597545 00000100:00000001:0.0:1788878985.722221:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878985.722223:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.722224:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878985.722226:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878985/real 1788878985] req@ffff94e511008e00 x1875765290118272/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878985.722231:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878985.722231:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878985.722233:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878985.722234:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878985.722235:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722237:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878985.722238:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878985.722240:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008e00 x1875765290118272/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878985.722244:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722245:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878985.722246:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722247:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878985.722248:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722249:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878985.722273:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878985.722274:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878985.722276:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878985.722277:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878985.722279:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290118272:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878985.722281:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878985.722282:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878985.722284:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878985.722285:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878985.722286:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878985.722288:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878985.722305:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290118400:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878985.722308:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878985.722309:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878985.722310:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878985.722311:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878985.722313:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.722316:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878985.722317:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508032 : -117763423043584 : ffff94e511008000) 00000100:00000001:0.0:1788878985.722320:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878985.722327:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878985.722336:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878985.722338:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878985.722339:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.722340:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878985.722342:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.722345:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304597002 00000100:00000001:0.0:1788878985.722347:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304597545 00000100:00000001:0.0:1788878985.722348:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304597002 00000100:00000001:0.0:1788878985.722350:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304597545 00000100:00000001:0.0:1788878985.722352:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878985.722353:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.722354:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878985.722357:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878985/real 1788878985] req@ffff94e511008000 x1875765290118400/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878985.722362:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878985.722363:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878985.722364:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878985.722365:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878985.722366:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722368:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878985.722369:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878985.722372:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008000 x1875765290118400/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878985.722378:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722379:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878985.722381:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722382:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878985.722382:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722384:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878985.722397:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878985.722398:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878985.722400:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878985.722402:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878985.722404:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290118400:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878985.722406:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722407:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878985.722408:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878985.722409:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878985.722411:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878985.722412:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878985.722413:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878985.722415:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878985.722416:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878985.722417:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878985.722418:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722420:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878985.722421:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878985.722422:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878985.722423:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878985.722424:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878985.722424:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878985.722437:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878985.722438:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878985.722439:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878985.722440:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722441:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878985.722444:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878985.722445:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878985.722446:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878985.722448:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878985.785020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878985.785027:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878985.785028:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878985.785032:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878985.785035:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878985.785038:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878985.785040:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878985.785042:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878985.785043:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878986.489067:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878986.489077:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878986.489085:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878986.489088:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878986.489168:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878986.489173:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878986.489177:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878986.809060:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878986.809069:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878986.809071:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878986.809078:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878986.809083:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878986.809088:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878986.809091:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878986.809094:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878986.809098:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878987.513086:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878987.513097:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878987.513104:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878987.513106:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878987.513247:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878987.513256:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878987.513262:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878987.833088:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878987.833097:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878987.833099:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878987.833107:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878987.833112:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878987.833116:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878987.833120:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878987.833123:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878987.833127:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878988.537098:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878988.537108:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878988.537116:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878988.537118:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878988.537149:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878988.537153:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878988.537157:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878988.857117:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878988.857127:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878988.857141:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878988.857149:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878988.857154:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878988.857159:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878988.857163:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878988.857167:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878988.857170:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878989.561082:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878989.561092:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878989.561100:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878989.561102:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878989.561131:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878989.561135:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878989.561139:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878989.881079:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878989.881088:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878989.881091:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878989.881098:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878989.881103:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878989.881108:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878989.881111:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878989.881115:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878989.881119:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878990.585031:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878990.585042:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878990.585049:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878990.585051:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878990.585168:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878990.585177:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878990.585185:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788878990.841073:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878990.841086:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788878990.841090:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788878990.841097:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878990.841103:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788878990.841115:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9929 00000100:00080000:1.0:1788878990.841131:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878990.841135:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878990.841138:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788878990.841151:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788878990.841155:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788878990.841161:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788878990.841164:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788878990.841167:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878990.841169:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788878990.841175:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878990.841179:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878990.841183:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788878990.841187:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878990.841192:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841204:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788878990.841207:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878990.841228:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788878990.841231:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841236:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788878990.841240:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788878990.841248:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841267:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878990.841280:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100ad80 x1875765290118528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878990.841295:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878990.841302:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841325:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788878990.841330:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878990.841335:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788878990.841337:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788878990.841339:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878990.841343:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788878990.841346:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9929 00000100:00080000:1.0:1788878990.841350:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878990.841352:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878990.841354:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788878990.841358:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788878990.841361:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788878990.841364:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788878990.841367:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788878990.841370:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878990.841374:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788878990.841378:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878990.841382:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878990.841385:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788878990.841394:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878990.841399:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1788878990.841399:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841408:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878990.841409:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00100000:1.0:1788878990.841415:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878990.841416:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878990.841424:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:1.0:1788878990.841425:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.841432:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841433:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878990.841446:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878990.841450:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878990.841453:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878990.841455:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878990.841469:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841469:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788878990.841474:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788878990.841480:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878990.841484:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290118528:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878990.841500:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.841505:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1788878990.841514:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008000 x1875765290118656/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788878990.841518:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878990.841526:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878990.841535:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788878990.841547:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1788878990.841551:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841552:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841565:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788878990.841569:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878990.841573:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788878990.841575:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788878990.841577:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878990.841581:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788878990.841584:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9929 00000100:00080000:1.0:1788878990.841588:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878990.841591:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878990.841594:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788878990.841594:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878990.841606:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286519680 : -117763423031936 : ffff94e51100ad80) 00000020:00000001:1.0:1788878990.841606:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788878990.841610:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788878990.841615:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788878990.841620:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788878990.841621:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788878990.841627:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878990.841632:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788878990.841636:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878990.841641:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878990.841644:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788878990.841647:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878990.841650:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841653:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788878990.841656:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788878990.841659:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841661:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878990.841668:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788878990.841670:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841673:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788878990.841675:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788878990.841677:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841685:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878990.841697:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008e00 x1875765290118784/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878990.841698:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878990.841714:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878990.841718:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841721:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.841726:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:1.0:1788878990.841727:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878990.841736:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878990.841739:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878990.841740:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788878990.841747:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878990.841748:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841752:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878990.841755:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:1.0:1788878990.841761:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9929 00000100:00000001:0.0:1788878990.841768:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788878990.841771:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788878990.841777:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304602665 00000100:00000001:1.0:1788878990.841787:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878990.841792:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788878990.841796:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788878990.841798:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788878990.841802:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:1.0:1788878990.841805:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788878990.841809:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788878990.841810:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878990.841820:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878990.841825:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304602665 00000100:00080000:1.0:1788878990.841831:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788878990.841837:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788878990.841840:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.841844:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878990.841851:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878990.841852:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.841860:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1788878990.841863:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788878990.841873:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878990/real 1788878990] req@ffff94e51100ad80 x1875765290118528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788878990.841875:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841888:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788878990.841895:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788878990.841944:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841948:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878990.841980:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788878990.841983:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.841986:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:0.0:1788878990.841989:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1788878990.841990:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:0.0:1788878990.841996:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1788878990.841998:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.842007:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788878990.842012:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.842015:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878990.842021:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878990.842027:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008700 x1875765290118912/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878990.842028:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878990.842045:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878990.842061:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100ad80 x1875765290118528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878990.842073:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878990.842077:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.842081:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.842117:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842126:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878990.842132:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842137:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878990.842139:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842144:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878990.842149:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878990.842154:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878990.842161:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878990.842165:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878990.842171:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290118528:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878990.842178:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842182:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878990.842184:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878990.842188:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878990.842206:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878990.842210:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878990.842214:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878990.842221:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878990.842224:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878990.842230:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878990.842232:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842237:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842251:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878990.842255:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.842260:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878990.842263:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878990.842267:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878990.842270:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878990.842273:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878990.842278:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878990.842282:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878990.842287:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878990.842295:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290118656:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878990.842301:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878990.842304:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878990.842307:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878990.842310:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878990.842314:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.842325:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878990.842327:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508032 : -117763423043584 : ffff94e511008000) 00000100:00000001:0.0:1788878990.842334:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878990.842356:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878990.842362:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878990.842365:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878990.842367:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.842370:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878990.842374:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.842379:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304602124 00000100:00000001:0.0:1788878990.842383:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304602665 00000100:00000001:0.0:1788878990.842387:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304602124 00000100:00000001:0.0:1788878990.842390:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304602665 00000100:00000001:0.0:1788878990.842393:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878990.842395:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.842398:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878990.842403:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878990/real 1788878990] req@ffff94e511008000 x1875765290118656/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878990.842413:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878990.842415:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878990.842419:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878990.842421:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878990.842423:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842425:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878990.842427:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878990.842432:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008000 x1875765290118656/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878990.842450:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842453:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878990.842456:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842458:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878990.842460:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842462:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878990.842483:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878990.842486:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878990.842489:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878990.842492:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878990.842497:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290118656:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878990.842505:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878990.842508:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878990.842511:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878990.842515:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878990.842519:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878990.842524:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878990.842530:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290118784:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878990.842536:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878990.842540:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878990.842543:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878990.842547:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878990.842553:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.842565:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878990.842568:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286511616 : -117763423040000 : ffff94e511008e00) 00000100:00000001:0.0:1788878990.842576:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878990.842601:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878990.842609:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878990.842613:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878990.842615:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.842618:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878990.842621:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.842627:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304602122 00000100:00000001:0.0:1788878990.842630:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304602665 00000100:00000001:0.0:1788878990.842633:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304602122 00000100:00000001:0.0:1788878990.842636:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304602665 00000100:00000001:0.0:1788878990.842639:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878990.842641:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.842644:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878990.842649:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878990/real 1788878990] req@ffff94e511008e00 x1875765290118784/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878990.842659:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878990.842660:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878990.842663:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878990.842665:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878990.842667:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842669:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878990.842671:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878990.842676:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008e00 x1875765290118784/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878990.842685:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842688:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878990.842691:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842692:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878990.842694:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842696:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878990.842715:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878990.842718:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878990.842721:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878990.842723:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878990.842728:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290118784:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878990.842733:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878990.842735:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878990.842737:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878990.842739:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878990.842742:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878990.842744:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878990.842747:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290118912:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878990.842751:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878990.842754:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878990.842756:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878990.842758:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878990.842762:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.842768:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878990.842770:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286509824 : -117763423041792 : ffff94e511008700) 00000100:00000001:0.0:1788878990.842775:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878990.842789:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878990.842794:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878990.842806:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878990.842808:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.842810:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878990.842813:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.842817:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304602122 00000100:00000001:0.0:1788878990.842820:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304602665 00000100:00000001:0.0:1788878990.842823:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304602122 00000100:00000001:0.0:1788878990.842826:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304602665 00000100:00000001:0.0:1788878990.842829:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878990.842831:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.842833:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878990.842838:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878990/real 1788878990] req@ffff94e511008700 x1875765290118912/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878990.842855:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878990.842857:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878990.842860:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878990.842862:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878990.842863:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842865:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878990.842867:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878990.842871:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008700 x1875765290118912/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878990.842879:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842881:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878990.842884:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842886:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878990.842887:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842889:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878990.842938:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878990.842942:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878990.842946:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878990.842948:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878990.842982:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290118912:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878990.842987:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.842990:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878990.842992:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878990.842995:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878990.842999:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878990.843001:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878990.843003:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878990.843006:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878990.843008:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878990.843011:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878990.843013:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.843016:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878990.843017:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878990.843030:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878990.843032:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878990.843034:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878990.843036:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878990.843039:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878990.843041:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878990.843045:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878990.843048:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.843051:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878990.843053:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878990.843056:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878990.843059:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878990.843061:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878990.843064:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878990.843067:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878990.843070:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878990.843072:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878990.843075:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.843079:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878990.843086:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878990.843089:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878990.843093:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878990.843096:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878990.905053:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878990.905061:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878990.905064:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878990.905071:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878990.905076:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878990.905081:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878990.905085:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878990.905088:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878990.905092:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878991.609006:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878991.609013:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878991.609018:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878991.609019:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878991.609092:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878991.609096:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878991.609099:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878991.929044:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878991.929057:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878991.929061:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878991.929069:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878991.929074:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878991.929080:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878991.929084:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878991.929098:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878991.929101:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878992.633075:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878992.633084:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878992.633090:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878992.633094:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878992.633097:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878992.633101:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878992.633105:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878992.633107:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878992.633109:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878992.633111:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878992.633151:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878992.633154:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878992.633158:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878992.953117:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878992.953141:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878992.953145:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878992.953152:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878992.953159:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878992.953164:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878992.953169:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878992.953173:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878992.953177:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878993.657043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878993.657052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878993.657058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878993.657059:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878993.657087:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878993.657090:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878993.657093:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878993.977106:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878993.977118:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878993.977121:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878993.977129:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878993.977135:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878993.977140:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878993.977144:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878993.977147:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878993.977151:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878994.681088:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878994.681101:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878994.681112:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878994.681115:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878994.681224:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878994.681232:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878994.681239:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878995.001098:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878995.001109:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878995.001112:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878995.001120:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878995.001126:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878995.001132:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878995.001135:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878995.001139:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878995.001143:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878995.705072:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878995.705086:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788878995.705090:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788878995.705096:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878995.705104:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788878995.705107:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878995.705110:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878995.705113:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878995.705118:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878995.705120:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878995.705230:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878995.705237:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878995.705242:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788878995.961087:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878995.961102:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788878995.961106:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788878995.961112:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878995.961119:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788878995.961130:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9934 00000100:00080000:1.0:1788878995.961146:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878995.961150:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878995.961153:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788878995.961165:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788878995.961169:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788878995.961174:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788878995.961177:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788878995.961180:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878995.961182:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788878995.961188:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878995.961191:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878995.961195:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788878995.961200:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878995.961204:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878995.961207:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788878995.961210:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878995.961229:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788878995.961231:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878995.961236:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788878995.961240:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788878995.961248:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878995.961268:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878995.961281:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100bb80 x1875765290119040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788878995.961296:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878995.961302:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878995.961329:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788878995.961333:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878995.961338:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788878995.961340:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788878995.961342:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878995.961345:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788878995.961348:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9934 00000100:00080000:1.0:1788878995.961352:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878995.961354:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878995.961355:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788878995.961359:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788878995.961360:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788878995.961363:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788878995.961366:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788878995.961369:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878995.961371:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788878995.961375:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878995.961379:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878995.961382:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878995.961390:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1788878995.961391:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878995.961399:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.961401:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878995.961402:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788878995.961408:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:1.0:1788878995.961412:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788878995.961414:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:1.0:1788878995.961417:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.961419:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878995.961422:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878995.961431:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788878995.961432:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788878995.961435:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878995.961439:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788878995.961440:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788878995.961445:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788878995.961448:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878995.961450:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878995.961459:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290119040:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788878995.961460:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.961473:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1788878995.961475:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100aa00 x1875765290119168/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788878995.961481:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878995.961486:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878995.961490:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788878995.961501:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1788878995.961502:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878995.961507:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878995.961514:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788878995.961517:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788878995.961520:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788878995.961522:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788878995.961524:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788878995.961527:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788878995.961530:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9934 00000100:00080000:1.0:1788878995.961534:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878995.961536:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878995.961537:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878995.961539:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:1.0:1788878995.961541:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788878995.961546:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788878995.961549:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286523264 : -117763423028352 : ffff94e51100bb80) 00000020:00000001:1.0:1788878995.961550:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788878995.961557:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788878995.961562:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878995.961566:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788878995.961567:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788878995.961575:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878995.961578:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878995.961580:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788878995.961583:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878995.961585:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878995.961588:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788878995.961590:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788878995.961593:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878995.961595:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878995.961601:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788878995.961603:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878995.961606:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788878995.961607:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788878995.961609:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878995.961615:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878995.961623:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b480 x1875765290119296/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1788878995.961641:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788878995.961641:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788878995.961649:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878995.961656:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788878995.961661:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788878995.961665:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:1.0:1788878995.961669:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788878995.961674:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788878995.961676:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788878995.961681:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788878995.961682:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788878995.961686:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788878995.961692:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788878995.961697:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788878995.961704:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9934 00000100:00000001:0.0:1788878995.961707:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304607785 00000100:00080000:1.0:1788878995.961715:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788878995.961720:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788878995.961722:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878995.961726:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:1.0:1788878995.961727:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788878995.961732:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788878995.961737:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:1.0:1788878995.961740:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788878995.961744:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788878995.961745:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304607785 00000100:00000001:1.0:1788878995.961749:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788878995.961757:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1788878995.961758:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788878995.961767:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.961769:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788878995.961770:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788878995.961777:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788878995.961778:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878995.961786:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1788878995.961788:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878995.961796:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:0.0:1788878995.961798:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878995/real 1788878995] req@ffff94e51100bb80 x1875765290119040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788878995.961802:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788878995.961814:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878995.961824:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788878995.961829:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878995.961834:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1788878995.961836:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788878995.961840:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:1.0:1788878995.961843:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.961846:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788878995.961850:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788878995.961851:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788878995.961857:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788878995.961859:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878995.961865:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788878995.961866:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878995.961873:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100bb80 x1875765290119040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788878995.961878:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.961893:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788878995.961894:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b800 x1875765290119424/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788878995.961966:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878995.961972:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.961975:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878995.961979:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.961986:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878995.961991:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878995.962004:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1788878995.962006:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00080000:0.0:1788878995.962013:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788878995.962017:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.962019:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1788878995.962028:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878995.962030:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290119040:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788878995.962044:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962047:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878995.962050:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878995.962052:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878995.962058:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878995.962060:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878995.962063:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878995.962068:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878995.962071:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878995.962075:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878995.962077:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962080:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962090:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878995.962093:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.962095:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878995.962097:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788878995.962099:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878995.962101:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878995.962103:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878995.962106:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878995.962108:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878995.962111:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878995.962114:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290119168:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878995.962118:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878995.962121:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878995.962123:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878995.962126:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878995.962131:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.962143:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878995.962145:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:0.0:1788878995.962153:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878995.962178:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878995.962187:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878995.962192:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878995.962194:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.962198:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878995.962203:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.962211:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304607244 00000100:00000001:0.0:1788878995.962216:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304607785 00000100:00000001:0.0:1788878995.962221:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304607244 00000100:00000001:0.0:1788878995.962225:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304607785 00000100:00000001:0.0:1788878995.962229:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878995.962233:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.962236:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878995.962244:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878995/real 1788878995] req@ffff94e51100aa00 x1875765290119168/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878995.962261:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878995.962263:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878995.962267:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878995.962269:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878995.962271:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962273:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878995.962275:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878995.962280:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290119168/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878995.962288:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962291:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878995.962294:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962296:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878995.962297:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962299:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878995.962319:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878995.962321:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878995.962326:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878995.962329:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878995.962333:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290119168:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788878995.962337:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878995.962339:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878995.962341:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878995.962344:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878995.962346:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878995.962349:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878995.962352:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290119296:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878995.962356:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878995.962358:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878995.962360:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878995.962362:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878995.962366:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.962373:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878995.962375:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:0.0:1788878995.962380:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878995.962395:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878995.962400:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878995.962403:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878995.962404:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.962406:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878995.962409:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.962413:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304607242 00000100:00000001:0.0:1788878995.962416:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304607785 00000100:00000001:0.0:1788878995.962419:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304607242 00000100:00000001:0.0:1788878995.962422:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304607785 00000100:00000001:0.0:1788878995.962424:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878995.962426:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.962428:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878995.962433:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878995/real 1788878995] req@ffff94e51100b480 x1875765290119296/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878995.962443:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878995.962445:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878995.962447:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878995.962449:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878995.962451:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962452:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878995.962454:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878995.962458:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290119296/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878995.962465:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962467:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878995.962470:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962471:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878995.962473:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962474:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878995.962492:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878995.962495:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878995.962499:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878995.962502:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878995.962508:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290119296:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878995.962513:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788878995.962516:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788878995.962518:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878995.962522:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788878995.962524:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788878995.962528:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788878995.962533:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290119424:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878995.962539:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788878995.962543:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788878995.962546:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788878995.962549:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788878995.962554:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.962564:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788878995.962567:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:0.0:1788878995.962573:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788878995.962594:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788878995.962600:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788878995.962603:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878995.962605:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.962607:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788878995.962610:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.962614:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304607242 00000100:00000001:0.0:1788878995.962617:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304607785 00000100:00000001:0.0:1788878995.962620:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304607242 00000100:00000001:0.0:1788878995.962622:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304607785 00000100:00000001:0.0:1788878995.962624:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788878995.962626:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.962628:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788878995.962633:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788878995/real 1788878995] req@ffff94e51100b800 x1875765290119424/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788878995.962641:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788878995.962643:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788878995.962645:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788878995.962647:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788878995.962649:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962651:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878995.962652:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788878995.962657:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290119424/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788878995.962664:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962667:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788878995.962670:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962672:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788878995.962674:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962676:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788878995.962693:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788878995.962696:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788878995.962700:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788878995.962703:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788878995.962706:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290119424:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788878995.962710:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962712:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878995.962713:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878995.962715:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878995.962717:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878995.962719:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878995.962721:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878995.962724:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878995.962725:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878995.962728:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878995.962729:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962741:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878995.962743:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878995.962744:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878995.962746:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878995.962748:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878995.962749:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878995.962751:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878995.962752:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878995.962754:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878995.962755:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962757:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788878995.962758:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788878995.962759:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788878995.962761:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788878995.962762:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788878995.962763:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788878995.962765:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788878995.962767:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788878995.962769:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788878995.962770:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962772:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788878995.962776:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788878995.962778:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788878995.962780:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788878995.962782:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788878996.025073:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878996.025083:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878996.025086:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878996.025091:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878996.025095:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878996.025100:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878996.025103:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878996.025106:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878996.025109:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878996.729056:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878996.729065:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878996.729071:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878996.729072:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878996.729102:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878996.729105:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878996.729108:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878997.049031:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878997.049039:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878997.049041:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878997.049047:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878997.049051:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878997.049056:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878997.049059:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878997.049063:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878997.049066:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878997.753017:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878997.753022:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878997.753026:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878997.753027:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788878997.753051:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788878997.753054:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788878997.753056:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878998.073031:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878998.073042:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878998.073045:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878998.073053:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878998.073058:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878998.073064:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878998.073069:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878998.073073:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878998.073076:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878998.777050:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878998.777059:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878998.777066:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878998.777068:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788878998.777179:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788878998.777188:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788878998.777195:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788878999.097023:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788878999.097034:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788878999.097038:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788878999.097046:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788878999.097052:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788878999.097058:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878999.097062:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788878999.097066:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788878999.097070:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878999.801043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878999.801055:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788878999.801067:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788878999.801070:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788878999.801180:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788878999.801190:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788878999.801197:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879000.121017:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879000.121028:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879000.121031:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879000.121040:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879000.121046:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879000.121051:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879000.121055:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879000.121058:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879000.121062:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879000.825105:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879000.825116:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879000.825123:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879000.825125:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879000.825154:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879000.825158:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879000.825162:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879001.081065:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879001.081081:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879001.081085:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879001.081090:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879001.081097:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879001.081106:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9939 00000100:00080000:1.0:1788879001.081121:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879001.081126:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879001.081129:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879001.081139:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879001.081143:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879001.081147:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879001.081150:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879001.081152:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879001.081153:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879001.081159:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879001.081161:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879001.081164:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879001.081168:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879001.081171:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879001.081173:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879001.081176:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879001.081189:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879001.081190:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879001.081193:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879001.081196:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879001.081201:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879001.081215:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879001.081225:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100a680 x1875765290119552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879001.081235:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879001.081240:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879001.081260:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879001.081263:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879001.081267:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879001.081268:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879001.081270:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879001.081273:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879001.081275:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9939 00000100:00080000:1.0:1788879001.081279:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879001.081281:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879001.081283:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879001.081287:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879001.081291:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879001.081294:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879001.081296:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879001.081299:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879001.081301:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879001.081308:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879001.081311:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879001.081313:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879001.081319:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879001.081322:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879001.081323:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879001.081329:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788879001.081330:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:1.0:1788879001.081332:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879001.081342:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.081344:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1788879001.081348:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788879001.081351:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1788879001.081352:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879001.081360:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788879001.081361:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788879001.081375:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788879001.081376:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879001.081383:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879001.081389:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788879001.081391:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879001.081396:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788879001.081407:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009f80 x1875765290119680/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788879001.081408:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879001.081431:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290119552:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879001.081436:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879001.081449:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.081454:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788879001.081458:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879001.081463:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1788879001.081464:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1788879001.081469:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879001.081472:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:0.0:1788879001.081477:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788879001.081479:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879001.081489:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879001.081493:0:178134:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:0.0:1788879001.081495:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879001.081503:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9939 00000100:00080000:1.0:1788879001.081509:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879001.081512:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879001.081513:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879001.081519:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879001.081521:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879001.081523:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879001.081525:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879001.081527:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879001.081529:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788879001.081531:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1788879001.081533:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879001.081536:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286517888 : -117763423033728 : ffff94e51100a680) 00000100:00000001:1.0:1788879001.081539:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879001.081551:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788879001.081553:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1788879001.081557:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879001.081561:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879001.081562:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879001.081565:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879001.081568:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879001.081570:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879001.081576:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879001.081577:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879001.081580:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879001.081583:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879001.081585:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879001.081590:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879001.081595:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009180 x1875765290119808/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879001.081603:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879001.081605:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879001.081608:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879001.081610:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879001.081614:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879001.081616:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879001.081617:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879001.081620:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879001.081622:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9939 00000100:00080000:1.0:1788879001.081626:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879001.081628:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879001.081630:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879001.081633:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879001.081636:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879001.081638:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000800:00000100:0.0:1788879001.081653:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:1.0:1788879001.081654:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879001.081668:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879001.081670:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879001.081676:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879001.081680:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879001.081682:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879001.081686:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879001.081690:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.081694:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879001.081694:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879001.081708:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788879001.081712:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00000001:1.0:1788879001.081713:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.081719:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879001.081719:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.081731:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:1.0:1788879001.081733:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879001.081737:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879001.081740:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879001.081742:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788879001.081743:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879001.081746:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.081758:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304612905 00000100:00000001:1.0:1788879001.081758:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879001.081776:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008380 x1875765290119936/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879001.081784:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879001.081786:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788879001.081787:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879001.081801:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.081809:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304612905 00000100:00000001:0.0:1788879001.081819:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788879001.081825:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879001.081829:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.081834:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788879001.081848:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879001/real 1788879001] req@ffff94e51100a680 x1875765290119552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788879001.081866:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879001.081868:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879001.081873:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879001.081878:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879001.081957:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.081964:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879001.081967:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879001.081979:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100a680 x1875765290119552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879001.081997:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082004:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879001.082011:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082015:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879001.082018:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082023:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879001.082027:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879001.082035:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879001.082041:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879001.082045:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879001.082050:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290119552:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879001.082058:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082061:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879001.082063:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879001.082067:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879001.082073:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879001.082077:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879001.082080:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879001.082086:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879001.082088:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879001.082092:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879001.082094:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082097:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082108:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879001.082111:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.082114:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879001.082116:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879001.082118:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879001.082121:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879001.082123:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879001.082127:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879001.082131:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879001.082133:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879001.082138:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290119680:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788879001.082142:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879001.082146:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879001.082148:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879001.082151:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879001.082155:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.082166:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879001.082168:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286516096 : -117763423035520 : ffff94e511009f80) 00000100:00000001:0.0:1788879001.082175:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788879001.082202:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879001.082212:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879001.082217:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879001.082220:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.082224:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879001.082229:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.082237:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304612364 00000100:00000001:0.0:1788879001.082244:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304612905 00000100:00000001:0.0:1788879001.082249:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304612364 00000100:00000001:0.0:1788879001.082254:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304612905 00000100:00000001:0.0:1788879001.082258:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879001.082263:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.082267:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788879001.082276:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879001/real 1788879001] req@ffff94e511009f80 x1875765290119680/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788879001.082291:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879001.082293:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879001.082297:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879001.082300:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879001.082302:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082305:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879001.082307:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879001.082312:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009f80 x1875765290119680/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879001.082322:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082325:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879001.082328:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082330:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879001.082332:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082334:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879001.082354:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879001.082358:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879001.082362:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879001.082365:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879001.082370:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290119680:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788879001.082375:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879001.082377:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879001.082379:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879001.082382:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879001.082385:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879001.082387:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879001.082391:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290119808:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879001.082396:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879001.082399:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879001.082401:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879001.082403:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879001.082407:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.082415:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879001.082417:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286512512 : -117763423039104 : ffff94e511009180) 00000100:00000001:0.0:1788879001.082423:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788879001.082440:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879001.082447:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879001.082450:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879001.082452:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.082454:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879001.082457:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.082462:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304612362 00000100:00000001:0.0:1788879001.082465:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304612905 00000100:00000001:0.0:1788879001.082468:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304612362 00000100:00000001:0.0:1788879001.082471:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304612905 00000100:00000001:0.0:1788879001.082474:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879001.082476:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.082478:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788879001.082483:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879001/real 1788879001] req@ffff94e511009180 x1875765290119808/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788879001.082492:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879001.082494:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879001.082496:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879001.082498:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879001.082500:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082502:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879001.082504:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879001.082508:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009180 x1875765290119808/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879001.082516:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082519:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879001.082521:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082523:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879001.082525:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082527:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879001.082544:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879001.082547:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879001.082552:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879001.082555:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879001.082559:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290119808:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879001.082564:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879001.082565:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879001.082567:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879001.082570:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879001.082572:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879001.082574:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879001.082578:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290119936:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879001.082582:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879001.082585:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879001.082588:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879001.082592:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879001.082597:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.082607:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879001.082610:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508928 : -117763423042688 : ffff94e511008380) 00000100:00000001:0.0:1788879001.082617:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788879001.082638:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879001.082646:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879001.082650:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879001.082653:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.082657:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879001.082662:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.082670:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304612362 00000100:00000001:0.0:1788879001.082676:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304612905 00000100:00000001:0.0:1788879001.082680:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304612362 00000100:00000001:0.0:1788879001.082684:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304612905 00000100:00000001:0.0:1788879001.082688:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879001.082692:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.082695:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788879001.082702:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879001/real 1788879001] req@ffff94e511008380 x1875765290119936/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788879001.082712:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879001.082714:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879001.082718:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879001.082721:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879001.082722:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082725:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879001.082726:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879001.082731:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008380 x1875765290119936/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879001.082740:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082743:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879001.082745:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082748:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879001.082749:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082752:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879001.082769:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879001.082772:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879001.082776:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879001.082779:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879001.082783:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290119936:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879001.082788:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082791:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879001.082793:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879001.082795:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879001.082798:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879001.082799:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879001.082801:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879001.082808:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879001.082810:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879001.082813:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879001.082815:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082817:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879001.082819:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879001.082821:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879001.082824:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879001.082826:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879001.082827:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879001.082829:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879001.082830:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879001.082832:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879001.082833:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082835:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879001.082837:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879001.082838:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879001.082839:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879001.082841:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879001.082842:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879001.082844:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879001.082845:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879001.082847:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879001.082849:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082851:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879001.082855:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879001.082857:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879001.082859:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879001.082861:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879001.145009:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879001.145016:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879001.145018:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879001.145022:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879001.145025:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879001.145028:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879001.145030:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879001.145033:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879001.145035:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879001.849026:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879001.849034:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879001.849040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879001.849041:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879001.849075:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879001.849082:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879001.849086:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879002.168975:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879002.168981:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879002.168983:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879002.168987:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879002.168990:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879002.168993:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879002.168995:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879002.168997:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879002.168999:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879002.872989:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879002.872994:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879002.872998:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879002.873000:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879002.873002:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879002.873005:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879002.873007:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879002.873008:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879002.873010:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879002.873012:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879002.873070:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879002.873075:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879002.873079:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879003.193078:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879003.193085:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879003.193088:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879003.193094:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879003.193109:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879003.193113:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879003.193116:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879003.193119:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879003.193122:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879003.897024:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879003.897029:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879003.897033:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879003.897034:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879003.897103:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879003.897108:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879003.897112:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879004.216964:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879004.216970:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879004.216971:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879004.216975:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879004.216977:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879004.216980:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879004.216982:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879004.216985:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879004.216987:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879004.920978:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879004.920984:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879004.921001:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879004.921003:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879004.921022:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879004.921024:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879004.921027:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879005.240974:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879005.240979:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879005.240981:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879005.240986:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879005.240988:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879005.240991:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879005.240993:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879005.240995:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879005.240997:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879005.944998:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879005.945005:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879005.945007:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879005.945010:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879005.945014:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879005.945015:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879005.945017:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879005.945018:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879005.945021:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879005.945022:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879005.945038:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879005.945040:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879005.945041:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879006.201019:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879006.201031:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879006.201035:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879006.201039:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879006.201045:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879006.201053:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9945 00000100:00080000:1.0:1788879006.201066:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879006.201071:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879006.201074:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879006.201084:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879006.201088:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879006.201093:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879006.201096:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879006.201099:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879006.201102:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879006.201108:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879006.201111:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879006.201115:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879006.201119:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879006.201123:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879006.201126:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879006.201130:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879006.201142:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879006.201145:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879006.201148:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879006.201151:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879006.201156:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879006.201171:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879006.201183:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009880 x1875765290120064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879006.201198:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879006.201203:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879006.201220:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879006.201224:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879006.201229:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879006.201231:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879006.201234:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879006.201238:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879006.201242:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9945 00000100:00000001:0.0:1788879006.201249:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1788879006.201249:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879006.201256:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879006.201256:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879006.201259:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788879006.201261:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879006.201266:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1788879006.201267:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788879006.201269:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879006.201273:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1788879006.201276:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1788879006.201278:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:0.0:1788879006.201280:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1788879006.201285:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:0.0:1788879006.201287:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879006.201294:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1788879006.201296:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290120064:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879006.201298:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788879006.201305:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879006.201311:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1788879006.201312:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:0.0:1788879006.201316:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879006.201319:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879006.201321:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879006.201327:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879006.201328:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879006.201335:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879006.201342:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879006.201345:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879006.201346:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879006.201350:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286514304 : -117763423037312 : ffff94e511009880) 00000008:00100000:1.0:1788879006.201351:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788879006.201355:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:1.0:1788879006.201359:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879006.201365:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879006.201373:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879006.201375:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879006.201378:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879006.201380:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879006.201383:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879006.201392:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788879006.201395:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1788879006.201405:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008380 x1875765290120192/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879006.201409:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879006.201415:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879006.201419:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879006.201423:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879006.201424:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879006.201430:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.201431:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.201440:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304618025 00000100:00000001:1.0:1788879006.201441:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879006.201447:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788879006.201451:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:1.0:1788879006.201453:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879006.201459:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788879006.201461:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304618025 00000100:00080000:1.0:1788879006.201462:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879006.201471:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879006.201471:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879006.201477:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9945 00000100:00000001:0.0:1788879006.201478:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879006.201484:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.201489:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1788879006.201489:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879006.201496:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1788879006.201498:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879006/real 1788879006] req@ffff94e511009880 x1875765290120064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879006.201500:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879006.201511:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879006.201515:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000001:0.0:1788879006.201518:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879006.201521:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:1.0:1788879006.201521:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879006.201527:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000020:00000001:1.0:1788879006.201528:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879006.201531:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879006.201535:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879006.201536:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879006.201541:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788879006.201542:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879006.201545:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788879006.201553:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:0.0:1788879006.201554:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009880 x1875765290120064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879006.201564:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.201573:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788879006.201573:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788879006.201580:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1788879006.201580:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879006.201589:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.201590:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.201597:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879006.201599:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879006.201600:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:1.0:1788879006.201604:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788879006.201605:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879006.201609:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:1.0:1788879006.201612:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879006.201614:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1788879006.201620:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879006.201623:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879006.201626:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:1.0:1788879006.201630:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:0.0:1788879006.201631:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290120064:192.168.203.129@tcp:250:kworker.0 02000000:00000001:1.0:1788879006.201637:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.201639:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.201644:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879006.201644:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788879006.201649:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879006.201650:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879006.201653:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879006.201656:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879006.201658:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879006.201663:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879006.201665:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:1.0:1788879006.201668:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879006.201669:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879006.201675:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879006.201677:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879006.201680:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.201684:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879006.201684:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009180 x1875765290120320/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879006.201694:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879006.201698:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879006.201702:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879006.201703:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879006.201707:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879006.201711:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879006.201711:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.201715:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879006.201719:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879006.201722:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788879006.201722:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879006.201726:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879006.201731:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879006.201731:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1788879006.201739:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290120192:192.168.203.129@tcp:38:kworker.0 00000100:00080000:1.0:1788879006.201743:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879006.201744:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788879006.201749:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:0.0:1788879006.201750:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879006.201755:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879006.201756:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1788879006.201757:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879006.201763:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879006.201765:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788879006.201771:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1788879006.201773:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9945 00000100:00000001:0.0:1788879006.201775:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508928 : -117763423042688 : ffff94e511008380) 00000100:00080000:1.0:1788879006.201782:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879006.201783:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879006.201789:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879006.201792:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879006.201798:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000100:0.0:1788879006.201800:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:1.0:1788879006.201802:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788879006.201808:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:1.0:1788879006.201808:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879006.201813:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1788879006.201815:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879006.201816:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.201823:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879006.201825:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879006.201827:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879006.201830:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788879006.201833:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304617483 00000100:00080000:1.0:1788879006.201839:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879006.201840:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304618025 00000100:00000001:1.0:1788879006.201864:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.201865:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304617483 02000000:00000001:1.0:1788879006.201872:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788879006.201874:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304618025 02000000:00000001:1.0:1788879006.201879:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879006.201880:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879006.201889:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879006.201889:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.201898:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1788879006.201898:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:0.0:1788879006.201937:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879006/real 1788879006] req@ffff94e511008380 x1875765290120192/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788879006.201940:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879006.201950:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879006.201957:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1788879006.201960:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879006.201964:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1788879006.201966:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788879006.201968:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:1.0:1788879006.201970:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.201972:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879006.201976:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1788879006.201977:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788879006.201981:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788879006.201982:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879006.201987:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788879006.201989:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879006.201993:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008380 x1875765290120192/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879006.201999:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.202001:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202007:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879006.202010:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202012:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879006.202015:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879006.202015:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009f80 x1875765290120448/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879006.202021:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879006.202034:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879006.202037:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879006.202038:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879006.202046:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1788879006.202048:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879006.202053:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879006.202059:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879006.202065:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290120192:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788879006.202071:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202075:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879006.202077:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879006.202079:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879006.202083:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879006.202085:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879006.202087:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879006.202091:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879006.202092:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879006.202095:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879006.202097:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202099:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202104:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879006.202106:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.202109:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879006.202111:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879006.202113:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879006.202115:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879006.202116:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879006.202119:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879006.202122:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879006.202125:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879006.202129:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290120320:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879006.202132:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879006.202134:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879006.202135:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879006.202137:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879006.202140:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.202146:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879006.202148:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286512512 : -117763423039104 : ffff94e511009180) 00000100:00000001:0.0:1788879006.202151:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788879006.202164:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879006.202168:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879006.202170:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879006.202172:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.202173:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879006.202176:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.202179:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304617482 00000100:00000001:0.0:1788879006.202181:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304618025 00000100:00000001:0.0:1788879006.202183:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304617482 00000100:00000001:0.0:1788879006.202185:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304618025 00000100:00000001:0.0:1788879006.202187:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879006.202188:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.202190:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788879006.202193:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879006/real 1788879006] req@ffff94e511009180 x1875765290120320/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788879006.202200:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879006.202201:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879006.202203:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879006.202204:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879006.202205:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202207:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879006.202208:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879006.202211:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009180 x1875765290120320/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879006.202217:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202219:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879006.202221:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202222:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879006.202223:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202225:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879006.202236:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879006.202239:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879006.202242:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879006.202244:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879006.202246:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290120320:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879006.202249:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879006.202250:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879006.202251:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879006.202253:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879006.202254:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879006.202256:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879006.202258:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290120448:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879006.202261:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879006.202263:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879006.202264:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879006.202265:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879006.202268:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.202271:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879006.202272:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286516096 : -117763423035520 : ffff94e511009f80) 00000100:00000001:0.0:1788879006.202275:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788879006.202282:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879006.202285:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879006.202287:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879006.202288:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.202289:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879006.202291:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.202294:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304617482 00000100:00000001:0.0:1788879006.202296:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304618025 00000100:00000001:0.0:1788879006.202298:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304617482 00000100:00000001:0.0:1788879006.202300:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304618025 00000100:00000001:0.0:1788879006.202303:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879006.202304:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.202306:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788879006.202309:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879006/real 1788879006] req@ffff94e511009f80 x1875765290120448/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788879006.202315:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879006.202316:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879006.202318:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879006.202319:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879006.202320:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202321:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879006.202323:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879006.202325:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009f80 x1875765290120448/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879006.202331:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202332:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879006.202334:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202336:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879006.202337:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202338:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879006.202349:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879006.202351:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879006.202354:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879006.202355:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879006.202358:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290120448:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879006.202361:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202363:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879006.202364:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879006.202365:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879006.202367:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879006.202368:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879006.202369:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879006.202371:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879006.202372:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879006.202374:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879006.202375:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202376:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879006.202377:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879006.202378:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879006.202380:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879006.202381:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879006.202382:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879006.202384:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879006.202385:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879006.202386:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879006.202387:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202388:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879006.202391:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879006.202392:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879006.202393:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879006.202395:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879006.264980:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879006.264986:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879006.264988:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879006.264992:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879006.264996:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879006.264999:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879006.265001:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879006.265003:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879006.265005:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879006.968982:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879006.968987:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879006.968991:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879006.968992:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879006.969049:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879006.969053:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879006.969056:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879007.288956:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879007.288961:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879007.288963:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879007.288967:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879007.288970:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879007.288974:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879007.288976:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879007.288978:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879007.288980:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879007.993036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879007.993043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879007.993050:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879007.993052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879007.993161:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879007.993167:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879007.993172:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879008.313027:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879008.313038:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879008.313042:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879008.313053:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879008.313060:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879008.313066:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879008.313071:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879008.313075:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879008.313081:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879009.017117:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879009.017128:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879009.017135:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879009.017137:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879009.017179:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879009.017185:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879009.017191:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879009.337072:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879009.337080:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879009.337083:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879009.337089:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879009.337094:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879009.337098:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879009.337101:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879009.337104:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879009.337107:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879010.041053:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879010.041064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879010.041073:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879010.041076:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879010.041108:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879010.041114:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879010.041117:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879010.361044:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879010.361057:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879010.361061:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879010.361071:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879010.361079:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879010.361087:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879010.361093:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879010.361098:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879010.361104:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879011.065016:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879011.065024:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879011.065030:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879011.065032:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879011.065177:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879011.065186:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879011.065193:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879011.321085:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879011.321102:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879011.321106:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879011.321112:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879011.321118:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879011.321129:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9950 00000100:00080000:0.0:1788879011.321146:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879011.321150:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879011.321153:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879011.321164:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879011.321168:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879011.321173:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879011.321177:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879011.321180:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879011.321182:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879011.321188:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879011.321191:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879011.321195:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879011.321200:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879011.321205:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879011.321208:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879011.321212:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879011.321231:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879011.321235:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879011.321241:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879011.321246:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879011.321255:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879011.321280:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879011.321300:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290120576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879011.321323:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879011.321329:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879011.321360:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879011.321365:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879011.321370:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879011.321372:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879011.321376:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879011.321381:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879011.321387:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9950 00000100:00080000:0.0:1788879011.321396:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879011.321399:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879011.321401:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879011.321407:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788879011.321408:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879011.321418:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879011.321420:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788879011.321424:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879011.321425:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788879011.321436:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879011.321437:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879011.321449:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879011.321451:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879011.321459:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879011.321462:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879011.321468:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1788879011.321469:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788879011.321483:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879011.321484:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879011.321503:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1788879011.321513:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290120576:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788879011.321518:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879011.321533:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.321533:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788879011.321543:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879011.321544:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879011.321549:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788879011.321551:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879011.321556:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:0.0:1788879011.321559:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879011.321564:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879011.321568:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879011.321574:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879011.321578:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879011.321581:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879011.321583:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879011.321587:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.321593:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879011.321597:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.321598:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 00000100:00000001:1.0:1788879011.321617:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1788879011.321625:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290120704/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879011.321638:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879011.321641:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879011.321649:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879011.321654:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879011.321661:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879011.321665:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879011.321668:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788879011.321678:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879011.321679:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879011.321690:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9950 00000100:00000001:1.0:1788879011.321694:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:0.0:1788879011.321704:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879011.321705:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879011.321712:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879011.321719:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879011.321720:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879011.321729:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879011.321730:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.321738:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304623145 00000020:00000001:0.0:1788879011.321745:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879011.321749:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879011.321751:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1788879011.321753:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879011.321761:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879011.321764:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879011.321768:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879011.321770:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304623145 00000100:00080000:0.0:1788879011.321776:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879011.321782:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.321784:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1788879011.321787:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879011.321796:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879011.321799:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879011.321802:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.321828:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.321833:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879011.321837:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879011.321843:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879011.321853:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879011.321855:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879011/real 1788879011] req@ffff94e508978380 x1875765290120576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879011.321866:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879011.321888:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1788879011.321889:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000400:00000001:1.0:1788879011.321891:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1788879011.321896:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.321899:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879011.321928:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879011.321931:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.321936:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879011.321938:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879011.321944:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290120576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879011.321952:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.321957:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879011.321960:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.321962:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879011.321964:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.321967:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879011.321972:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879011.321972:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1788879011.321979:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00080000:1.0:1788879011.321983:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:0.0:1788879011.321985:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879011.321992:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879011.321995:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1788879011.322000:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879011.322002:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290120576:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879011.322014:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879011.322017:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979180 x1875765290120832/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879011.322019:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879011.322023:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879011.322030:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879011.322041:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1788879011.322049:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000020:00000001:1.0:1788879011.322051:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:0.0:1788879011.322057:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879011.322059:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:0.0:1788879011.322063:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879011.322068:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879011.322070:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879011.322076:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:0.0:1788879011.322077:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879011.322085:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879011.322089:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00080000:0.0:1788879011.322090:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879011.322099:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879011.322100:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879011.322113:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9950 00000100:00000001:1.0:1788879011.322116:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879011.322122:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879011.322126:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879011.322128:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879011.322133:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879011.322135:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879011.322138:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879011.322139:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879011.322143:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879011.322145:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879011.322149:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879011.322151:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879011.322155:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879011.322156:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879011.322164:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879011.322169:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788879011.322170:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879011.322179:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879011.322181:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879011.322187:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879011.322187:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879011.322205:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879011.322207:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879011.322211:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879011.322219:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879011.322221:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788879011.322226:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879011.322236:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290120704:192.168.203.129@tcp:38:kworker.0 00000008:00000001:0.0:1788879011.322237:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879011.322247:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.322254:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879011.322263:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879011.322267:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879011.322268:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879011.322279:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879011.322281:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879011.322288:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879011.322291:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879011.322297:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879011.322299:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879011.322303:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 02000000:00000001:0.0:1788879011.322304:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.322309:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879011.322322:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879011.322334:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290120960/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788879011.322344:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879011.322358:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879011.322365:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879011.322367:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879011.322370:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879011.322373:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.322378:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879011.322385:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.322387:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.322396:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304622604 00000100:00000001:1.0:1788879011.322400:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304623145 00000100:00000001:1.0:1788879011.322403:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304622604 00000100:00000001:1.0:1788879011.322406:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304623145 00000100:00000001:1.0:1788879011.322408:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879011.322411:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.322412:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879011.322420:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879011/real 1788879011] req@ffff94e50897b800 x1875765290120704/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879011.322430:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879011.322432:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879011.322434:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879011.322437:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879011.322438:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322440:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879011.322441:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879011.322445:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290120704/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879011.322452:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322454:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879011.322457:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322458:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879011.322459:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322461:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879011.322472:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879011.322474:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879011.322478:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879011.322479:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879011.322482:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290120704:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879011.322485:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879011.322487:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879011.322488:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879011.322491:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879011.322493:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879011.322495:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879011.322498:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290120832:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879011.322501:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879011.322502:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879011.322504:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879011.322506:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879011.322509:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.322513:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879011.322514:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 00000100:00000001:1.0:1788879011.322518:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879011.322527:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879011.322530:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879011.322532:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879011.322534:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.322536:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879011.322538:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.322541:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304622602 00000100:00000001:1.0:1788879011.322543:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304623145 00000100:00000001:1.0:1788879011.322546:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304622602 00000100:00000001:1.0:1788879011.322548:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304623145 00000100:00000001:1.0:1788879011.322550:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879011.322551:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.322553:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879011.322556:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879011/real 1788879011] req@ffff94e508979180 x1875765290120832/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879011.322563:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879011.322564:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879011.322566:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879011.322567:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879011.322568:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322570:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879011.322571:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879011.322574:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290120832/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879011.322580:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322582:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879011.322584:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322585:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879011.322586:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322588:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879011.322597:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879011.322599:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879011.322601:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879011.322603:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879011.322606:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290120832:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879011.322610:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322612:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879011.322614:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879011.322615:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879011.322617:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879011.322618:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879011.322619:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879011.322622:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879011.322623:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879011.322626:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879011.322627:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322628:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879011.322630:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879011.322631:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879011.322632:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879011.322633:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879011.322634:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879011.322636:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879011.322636:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879011.322639:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879011.322640:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322641:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322645:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879011.322646:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.322648:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879011.322650:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879011.322652:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879011.322653:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879011.322655:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879011.322657:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879011.322658:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879011.322660:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879011.322662:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290120960:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879011.322666:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879011.322667:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879011.322669:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879011.322671:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879011.322673:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.322677:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879011.322678:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 00000100:00000001:1.0:1788879011.322681:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879011.322689:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879011.322693:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879011.322694:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879011.322696:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.322697:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879011.322699:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.322703:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304622602 00000100:00000001:1.0:1788879011.322705:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304623145 00000100:00000001:1.0:1788879011.322707:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304622602 00000100:00000001:1.0:1788879011.322710:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304623145 00000100:00000001:1.0:1788879011.322711:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879011.322713:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.322714:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879011.322718:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879011/real 1788879011] req@ffff94e508978380 x1875765290120960/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879011.322724:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879011.322725:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879011.322727:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879011.322729:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879011.322730:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322732:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879011.322733:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879011.322736:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290120960/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879011.322742:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322743:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879011.322745:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322747:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879011.322748:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322750:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879011.322761:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879011.322766:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879011.322770:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879011.322774:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879011.322778:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290120960:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879011.322783:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322787:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879011.322788:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879011.322790:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879011.322793:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879011.322795:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879011.322797:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879011.322800:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879011.322802:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879011.322804:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879011.322806:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322825:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.322829:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879011.322831:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.322834:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879011.322836:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879011.384997:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879011.385003:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879011.385006:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879011.385011:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879011.385017:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879011.385021:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.385024:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879011.385027:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879011.385030:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879012.089046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879012.089053:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879012.089058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879012.089062:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879012.089065:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879012.089068:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879012.089070:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879012.089072:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879012.089073:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879012.089075:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879012.089222:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879012.089230:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879012.089235:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879012.409015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879012.409024:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879012.409027:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879012.409033:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879012.409037:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879012.409041:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879012.409044:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879012.409047:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879012.409050:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879013.113025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879013.113034:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879013.113040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879013.113042:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879013.113066:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879013.113070:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879013.113073:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879013.433031:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879013.433042:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879013.433045:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879013.433053:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879013.433059:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879013.433065:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879013.433070:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879013.433075:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879013.433080:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879014.137072:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879014.137080:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879014.137087:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879014.137089:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879014.137133:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879014.137136:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879014.137139:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879014.457048:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879014.457057:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879014.457060:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879014.457066:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879014.457071:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879014.457075:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879014.457079:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879014.457082:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879014.457086:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879015.161056:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879015.161067:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879015.161070:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879015.161074:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879015.161081:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879015.161084:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879015.161086:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879015.161089:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879015.161094:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879015.161096:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879015.161229:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879015.161235:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879015.161240:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879015.481094:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879015.481103:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879015.481106:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879015.481113:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879015.481118:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879015.481123:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879015.481126:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879015.481129:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879015.481132:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879016.185031:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879016.185039:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879016.185045:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879016.185047:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879016.185156:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879016.185161:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879016.185165:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879016.441057:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879016.441066:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879016.441068:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879016.441071:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879016.441075:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879016.441081:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9955 00000100:00080000:0.0:1788879016.441089:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879016.441091:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879016.441093:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879016.441105:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879016.441109:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879016.441113:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879016.441116:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879016.441119:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879016.441121:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879016.441126:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879016.441129:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879016.441132:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879016.441136:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879016.441139:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879016.441142:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879016.441145:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879016.441155:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879016.441157:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879016.441163:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879016.441166:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879016.441171:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879016.441182:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879016.441192:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290121088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879016.441203:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879016.441209:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879016.441224:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879016.441228:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879016.441231:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879016.441233:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879016.441235:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879016.441238:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879016.441241:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9955 00000100:00080000:0.0:1788879016.441245:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879016.441248:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879016.441249:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879016.441253:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879016.441255:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879016.441257:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879016.441260:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879016.441261:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879016.441263:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879016.441267:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879016.441268:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788879016.441271:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879016.441274:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879016.441275:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441278:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788879016.441280:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879016.441282:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879016.441287:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879016.441289:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879016.441293:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879016.441294:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879016.441296:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879016.441298:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788879016.441303:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879016.441307:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290121088:192.168.203.129@tcp:250:kworker.0 00000008:00000001:0.0:1788879016.441309:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441312:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788879016.441313:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879016.441317:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879016.441320:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879016.441321:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879016.441322:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879016.441324:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879016.441328:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879016.441330:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879016.441334:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879016.441341:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879016.441349:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879016.441351:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290121216/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879016.441377:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879016.441381:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441385:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879016.441390:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:0.0:1788879016.441397:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441397:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788879016.441405:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879016.441410:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879016.441413:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879016.441416:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879016.441421:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879016.441442:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9955 00000100:00080000:0.0:1788879016.441447:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879016.441449:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879016.441450:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879016.441455:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879016.441456:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879016.441458:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879016.441460:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879016.441462:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879016.441463:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879016.441466:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879016.441468:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879016.441470:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879016.441472:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879016.441474:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879016.441477:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304628265 02000000:00000001:0.0:1788879016.441478:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879016.441482:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879016.441484:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879016.441487:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441487:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788879016.441510:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441517:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304628265 02000000:00000001:0.0:1788879016.441519:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879016.441525:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441527:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788879016.441530:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879016.441533:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879016.441535:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879016.441538:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879016.441541:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441543:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441550:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879016.441557:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441559:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879016.441566:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788879016.441573:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788879016.441574:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290121344/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879016.441578:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879016.441581:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879016.441584:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879016.441587:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879016.441594:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879016.441600:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879016.441601:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879016.441604:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879016.441609:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879016.441613:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290121216:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879016.441619:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788879016.441622:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879016.441622:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879016.441626:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879016.441631:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879016.441631:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879016.441640:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879016.441641:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879016.441647:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879016.441652:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879016.441656:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879016.441662:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9955 00000100:00080000:0.0:1788879016.441669:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879016.441672:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879016.441674:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879016.441681:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879016.441682:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879016.441685:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879016.441689:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879016.441691:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879016.441694:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879016.441698:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879016.441701:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879016.441703:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879016.441707:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879016.441709:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879016.441711:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879016.441713:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879016.441715:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879016.441717:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441719:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1788879016.441723:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879016.441724:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 02000000:00000001:0.0:1788879016.441727:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879016.441731:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879016.441734:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879016.441736:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441741:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879016.441743:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879016.441751:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290121472/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879016.441759:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879016.441761:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441763:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879016.441764:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441795:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304627723 00000100:00000001:1.0:1788879016.441817:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304628265 00000100:00000001:1.0:1788879016.441819:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304627723 00000100:00000001:1.0:1788879016.441821:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304628265 00000100:00000001:1.0:1788879016.441823:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879016.441825:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441826:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441828:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879016.441831:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879016.441833:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788879016.441835:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879016.441837:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879016.441838:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879016.441839:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879016.441841:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879016.441843:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879016.441845:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879016.441847:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879016.441852:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290121344:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879016.441856:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879016.441858:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879016.441859:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879016.441861:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879016.441880:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441887:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879016.441888:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:1.0:1788879016.441892:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879016.441903:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441907:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304627721 00000100:00000001:1.0:1788879016.441910:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304628265 00000100:00000001:1.0:1788879016.441912:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304627721 00000100:00000001:1.0:1788879016.441915:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304628265 00000100:00000001:1.0:1788879016.441917:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879016.441919:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.441921:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879016.441923:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879016.441925:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879016.441928:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879016.441974:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879016.441977:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879016.441982:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290121472:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879016.441986:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879016.441988:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879016.441990:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879016.441992:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879016.442010:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.442016:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879016.442016:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:1.0:1788879016.442019:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879016.442028:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.442032:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304627722 00000100:00000001:1.0:1788879016.442036:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304628265 00000100:00000001:1.0:1788879016.442038:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304627722 00000100:00000001:1.0:1788879016.442041:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304628265 00000100:00000001:1.0:1788879016.442044:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879016.442046:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.442050:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.442052:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879016.442056:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879016.442058:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788879016.442060:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879016.442062:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879016.442065:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.442067:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879016.505008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879016.505014:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879016.505015:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879016.505030:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879016.505033:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879016.505036:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879016.505038:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879016.505040:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879016.505042:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879017.209066:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879017.209077:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879017.209084:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879017.209087:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879017.209119:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879017.209123:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879017.209127:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879017.529038:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879017.529047:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879017.529050:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879017.529057:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879017.529061:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879017.529066:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879017.529069:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879017.529073:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879017.529077:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879018.233101:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879018.233124:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879018.233131:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879018.233133:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879018.233170:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879018.233173:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879018.233177:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879018.553017:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879018.553025:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879018.553028:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879018.553035:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879018.553042:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879018.553048:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879018.553053:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879018.553057:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879018.553062:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879019.257104:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879019.257127:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879019.257135:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879019.257137:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879019.257292:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879019.257299:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879019.257304:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788879019.513288:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:2.0:1788879019.513346:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788879019.513370:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788879019.513391:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879019.513415:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879019.513419:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879019.513422:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:2.0:1788879019.513435:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788879019.513441:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879019.513444:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879019.513446:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879019.513448:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:2.0:1788879019.513453:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788879019.513458:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879019.513461:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879019.513464:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879019.513468:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:2.0:1788879019.513476:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788879019.513485:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879019.513490:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879019.513492:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879019.513495:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879019.513511:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879019.513518:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879019.513522:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879019.513536:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879016/real 1788879019] req@ffff94e50897bb80 x1875765290121344/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879019.513555:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879019.513558:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879019.513564:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879019.513568:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879019.513570:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.513574:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879019.513576:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879019.513583:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290121344/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879019.513592:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.513597:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879019.513601:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.513603:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879019.513605:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.513607:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879019.513628:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879019.513635:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879019.513644:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879019.513649:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879019.513657:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290121344:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879019.513664:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879019.513672:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879016/real 1788879019] req@ffff94e508978e00 x1875765290121472/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879019.513688:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879019.513690:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879019.513695:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879019.513698:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879019.513701:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.513706:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879019.513709:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879019.513728:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290121472/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879019.513741:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.513746:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879019.513751:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.513778:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879019.513780:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.513784:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879019.513804:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879019.513809:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879019.513816:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879019.513820:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879019.513827:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290121472:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879019.513835:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879019.513843:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879016/real 1788879019] req@ffff94e50897ad80 x1875765290121216/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879019.513876:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879019.513878:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879019.513883:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879019.513886:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879019.513888:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.513891:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879019.513892:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879019.513898:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290121216/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879019.513908:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.513911:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879019.513914:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.513955:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879019.513958:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.513961:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879019.514000:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879019.514005:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879019.514011:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879019.514015:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879019.514020:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290121216:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879019.514027:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879019.514041:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879016/real 1788879019] req@ffff94e508979c00 x1875765290121088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879019.514056:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879019.514059:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879019.514066:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879019.514071:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879019.514075:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.514078:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879019.514080:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879019.514088:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290121088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879019.514100:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.514103:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879019.514106:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.514109:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879019.514111:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.514114:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879019.514116:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879019.514120:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879019.514125:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879019.514128:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879019.514135:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290121088:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879019.514141:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.514145:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879019.514148:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879019.514151:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879019.514157:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879019.514161:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879019.514164:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879019.514169:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879019.514172:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879019.514178:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879019.514180:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.514183:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879019.514185:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879019.514187:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879019.514190:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879019.514192:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879019.514194:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879019.514197:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879019.514199:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879019.514203:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879019.514205:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.514208:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879019.514210:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879019.514211:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879019.514214:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879019.514216:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879019.514217:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879019.514221:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879019.514222:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879019.514226:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879019.514245:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.514249:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879019.514251:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879019.514253:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879019.514256:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879019.514259:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879019.514261:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879019.514264:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879019.514266:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879019.514269:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879019.514272:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.514275:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.514286:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879019.514289:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879019.514293:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879019.514313:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879019.577025:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879019.577035:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879019.577038:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879019.577047:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879019.577075:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879019.577079:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.577083:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879019.577087:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879019.577090:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879020.281022:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879020.281030:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879020.281035:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879020.281037:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879020.281155:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879020.281160:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879020.281165:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879020.601044:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879020.601052:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879020.601054:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879020.601060:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879020.601064:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879020.601069:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879020.601072:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879020.601074:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879020.601077:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879021.305032:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879021.305040:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879021.305047:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879021.305049:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879021.305074:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879021.305078:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879021.305081:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879021.561123:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879021.561135:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879021.561138:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879021.561143:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879021.561149:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879021.561158:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9960 00000100:00080000:2.0:1788879021.561171:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879021.561175:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879021.561178:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879021.561187:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879021.561190:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879021.561195:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879021.561198:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879021.561201:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879021.561203:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879021.561208:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879021.561212:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879021.561216:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879021.561220:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879021.561224:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879021.561228:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879021.561231:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879021.561243:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879021.561246:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879021.561249:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879021.561253:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879021.561258:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879021.561270:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879021.561283:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290121600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879021.561296:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879021.561301:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879021.561325:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879021.561330:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879021.561334:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879021.561337:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879021.561340:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879021.561344:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879021.561348:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9960 00000100:00080000:2.0:1788879021.561353:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879021.561357:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879021.561359:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879021.561363:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879021.561365:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879021.561368:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879021.561371:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879021.561374:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879021.561375:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879021.561378:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788879021.561381:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879021.561385:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879021.561388:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.561391:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788879021.561393:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879021.561397:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879021.561401:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879021.561403:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879021.561409:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879021.561410:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879021.561412:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879021.561416:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:2.0:1788879021.561419:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879021.561422:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788879021.561426:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879021.561432:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879021.561434:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290121600:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879021.561439:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879021.561444:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879021.561444:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879021.561450:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879021.561452:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879021.561453:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879021.561458:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879021.561459:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879021.561464:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879021.561470:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879021.561473:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879021.561481:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4000 x1875765290121728/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879021.561494:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879021.561494:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879021.561498:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.561500:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074559872 : -117763634991744 : ffff94e5045e6d80) 00000100:00000001:2.0:1788879021.561507:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.561508:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788879021.561515:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879021.561522:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879021.561524:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879021.561526:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879021.561530:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879021.561533:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9960 00000100:00080000:2.0:1788879021.561538:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879021.561541:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879021.561543:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879021.561548:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879021.561550:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879021.561554:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000800:00000100:1.0:1788879021.561558:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:2.0:1788879021.561560:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879021.561565:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879021.561568:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879021.561575:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788879021.561575:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879021.561582:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.561583:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788879021.561588:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879021.561590:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879021.561595:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879021.561596:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:2.0:1788879021.561602:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.561606:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879021.561606:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879021.561611:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879021.561616:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304633385 00000008:00000001:2.0:1788879021.561617:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879021.561625:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879021.561632:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879021.561633:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1788879021.561635:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879021.561641:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879021.561643:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879021.561645:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879021.561652:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.561657:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304633385 00000100:00100000:2.0:1788879021.561659:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290121856/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879021.561668:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879021.561677:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879021.561679:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879021.561684:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.561687:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879021.561691:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.561694:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1788879021.561698:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788879021.561707:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879021/real 1788879021] req@ffff94e5045e6d80 x1875765290121600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879021.561708:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879021.561713:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879021.561717:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879021.561727:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879021.561735:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9960 00000400:00000001:1.0:1788879021.561751:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879021.561755:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1788879021.561761:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879021.561764:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879021.561766:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879021.561771:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879021.561772:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879021.561776:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788879021.561780:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879021.561783:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1788879021.561785:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879021.561788:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788879021.561790:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879021.561797:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:1.0:1788879021.561801:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6d80 x1875765290121600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879021.561802:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879021.561806:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879021.561818:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879021.561828:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879021.561828:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879021.561837:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879021.561838:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788879021.561841:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879021.561846:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879021.561848:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.561854:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879021.561855:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879021.561859:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879021.561863:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1788879021.561866:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.561870:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879021.561872:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.561875:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788879021.561881:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00080000:1.0:1788879021.561883:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1788879021.561886:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879021.561891:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:1.0:1788879021.561894:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 02000000:00000001:2.0:1788879021.561895:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879021.561941:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879021.561949:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290121600:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879021.561950:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.561961:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879021.561963:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.561966:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879021.561971:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879021.561975:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:2.0:1788879021.561977:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290121984/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879021.561981:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879021.561986:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879021.561990:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879021.561997:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1788879021.561998:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879021.562001:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879021.562007:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.562009:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879021.562012:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.562013:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562021:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562030:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879021.562032:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.562035:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879021.562037:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879021.562039:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879021.562042:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879021.562044:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879021.562047:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879021.562050:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879021.562053:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879021.562058:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290121728:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879021.562063:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879021.562066:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879021.562067:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879021.562070:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879021.562074:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.562083:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879021.562092:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074548224 : -117763635003392 : ffff94e5045e4000) 00000100:00000001:1.0:1788879021.562099:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879021.562121:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879021.562128:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879021.562131:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879021.562133:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.562136:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879021.562139:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.562145:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304632844 00000100:00000001:1.0:1788879021.562148:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304633385 00000100:00000001:1.0:1788879021.562151:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304632844 00000100:00000001:1.0:1788879021.562154:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304633385 00000100:00000001:1.0:1788879021.562156:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879021.562159:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.562161:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879021.562167:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879021/real 1788879021] req@ffff94e5045e4000 x1875765290121728/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879021.562178:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879021.562179:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879021.562182:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879021.562184:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879021.562186:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562188:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879021.562198:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879021.562204:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4000 x1875765290121728/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879021.562217:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562220:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879021.562223:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562225:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879021.562227:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562229:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879021.562250:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879021.562253:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879021.562257:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879021.562260:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879021.562264:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290121728:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879021.562269:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879021.562270:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879021.562272:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879021.562275:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879021.562277:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879021.562279:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879021.562282:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290121856:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879021.562286:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879021.562289:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879021.562290:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879021.562293:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879021.562297:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.562304:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879021.562306:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:1.0:1788879021.562310:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879021.562327:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879021.562334:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879021.562337:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879021.562339:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.562341:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879021.562344:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.562348:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304632842 00000100:00000001:1.0:1788879021.562352:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304633385 00000100:00000001:1.0:1788879021.562355:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304632842 00000100:00000001:1.0:1788879021.562357:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304633385 00000100:00000001:1.0:1788879021.562361:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879021.562363:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.562365:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879021.562371:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879021/real 1788879021] req@ffff94e5045e4e00 x1875765290121856/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879021.562382:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879021.562384:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879021.562387:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879021.562389:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879021.562391:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562393:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879021.562395:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879021.562400:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290121856/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879021.562408:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562411:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879021.562414:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562416:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879021.562417:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562420:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879021.562436:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879021.562440:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879021.562444:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879021.562447:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879021.562451:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290121856:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879021.562455:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879021.562457:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879021.562458:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879021.562461:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879021.562463:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879021.562466:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879021.562470:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290121984:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879021.562474:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879021.562477:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879021.562479:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879021.562482:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879021.562486:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.562492:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879021.562494:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00000001:1.0:1788879021.562498:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879021.562514:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879021.562520:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879021.562523:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879021.562525:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.562527:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879021.562530:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.562535:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304632842 00000100:00000001:1.0:1788879021.562538:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304633385 00000100:00000001:1.0:1788879021.562541:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304632842 00000100:00000001:1.0:1788879021.562544:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304633385 00000100:00000001:1.0:1788879021.562547:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879021.562550:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.562552:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879021.562557:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879021/real 1788879021] req@ffff94e5045e7b80 x1875765290121984/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879021.562567:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879021.562568:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879021.562571:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879021.562573:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879021.562575:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562578:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879021.562580:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879021.562585:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290121984/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879021.562600:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562603:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879021.562606:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562608:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879021.562610:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562613:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879021.562629:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879021.562633:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879021.562638:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879021.562641:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879021.562645:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290121984:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879021.562650:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562653:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879021.562655:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879021.562657:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879021.562660:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879021.562662:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879021.562664:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879021.562667:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879021.562676:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879021.562680:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879021.562682:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562684:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879021.562686:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879021.562688:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879021.562690:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879021.562692:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879021.562694:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879021.562698:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879021.562699:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879021.562702:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879021.562703:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562705:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879021.562707:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879021.562709:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879021.562711:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879021.562713:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879021.562714:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879021.562716:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879021.562717:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879021.562719:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879021.562721:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562724:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.562729:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879021.562731:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.562733:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879021.562736:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879021.624991:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879021.624998:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879021.625000:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879021.625006:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879021.625011:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879021.625015:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.625018:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879021.625021:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879021.625025:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879022.329077:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879022.329088:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879022.329095:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879022.329100:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879022.329104:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879022.329109:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879022.329113:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879022.329116:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879022.329119:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879022.329122:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879022.329152:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879022.329155:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879022.329160:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879022.649040:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879022.649049:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879022.649052:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879022.649058:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879022.649063:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879022.649068:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879022.649071:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879022.649074:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879022.649078:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879023.353023:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879023.353047:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879023.353052:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879023.353054:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879023.353167:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879023.353175:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879023.353181:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879023.673093:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879023.673103:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879023.673106:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879023.673114:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879023.673120:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879023.673126:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879023.673130:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879023.673135:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879023.673138:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879024.377058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879024.377064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879024.377068:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879024.377069:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879024.377121:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879024.377125:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879024.377128:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879024.697042:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879024.697050:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879024.697053:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879024.697059:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879024.697064:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879024.697068:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879024.697071:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879024.697073:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879024.697076:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879025.401078:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879025.401087:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879025.401090:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879025.401106:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879025.401112:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879025.401115:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879025.401117:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879025.401120:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879025.401124:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879025.401126:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879025.401154:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879025.401157:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879025.401160:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879025.721096:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879025.721118:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879025.721121:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879025.721128:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879025.721133:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879025.721137:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879025.721141:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879025.721144:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879025.721148:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879026.425086:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879026.425097:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879026.425107:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879026.425110:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879026.425145:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879026.425148:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879026.425152:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879026.681100:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879026.681116:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879026.681121:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879026.681128:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879026.681135:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879026.681146:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9965 00000100:00080000:1.0:1788879026.681163:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879026.681169:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879026.681172:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879026.681185:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879026.681189:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879026.681209:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879026.681215:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879026.681220:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879026.681223:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879026.681232:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879026.681238:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879026.681243:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879026.681248:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879026.681252:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681255:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879026.681258:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879026.681273:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879026.681275:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681279:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879026.681283:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879026.681289:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681303:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879026.681315:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100a680 x1875765290122112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879026.681329:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879026.681335:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681363:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879026.681368:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879026.681373:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879026.681375:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879026.681377:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879026.681381:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879026.681384:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9965 00000100:00080000:1.0:1788879026.681390:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879026.681393:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879026.681394:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879026.681399:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879026.681401:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879026.681403:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879026.681407:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879026.681409:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879026.681411:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879026.681415:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879026.681418:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879026.681420:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879026.681427:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879026.681429:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681431:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879026.681436:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879026.681440:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681443:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879026.681447:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879026.681450:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681452:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879026.681455:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879026.681457:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681464:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879026.681471:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b800 x1875765290122240/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879026.681479:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879026.681487:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879026.681489:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879026.681493:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.681496:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879026.681502:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879026.681503:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.681507:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879026.681510:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1788879026.681518:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1788879026.681518:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879026.681521:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:0.0:1788879026.681525:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1788879026.681526:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1788879026.681537:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681538:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879026.681547:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9965 00000100:00080000:1.0:1788879026.681552:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879026.681555:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1788879026.681557:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290122112:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879026.681558:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879026.681567:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879026.681571:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879026.681576:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879026.681580:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:1.0:1788879026.681582:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879026.681589:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879026.681591:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879026.681597:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879026.681599:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879026.681601:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879026.681605:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879026.681607:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681609:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:0.0:1788879026.681612:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:1.0:1788879026.681613:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879026.681620:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879026.681622:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681623:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879026.681636:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879026.681637:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879026.681644:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681646:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879026.681648:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879026.681650:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879026.681652:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681658:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879026.681679:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b480 x1875765290122368/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879026.681689:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879026.681692:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879026.681693:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.681700:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286517888 : -117763423033728 : ffff94e51100a680) 00000100:00000001:1.0:1788879026.681726:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879026.681736:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1788879026.681737:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1788879026.681745:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879026.681749:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879026.681751:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879026.681755:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879026.681761:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9965 00000100:00080000:1.0:1788879026.681782:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879026.681786:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879026.681788:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:0.0:1788879026.681797:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:1.0:1788879026.681797:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879026.681804:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879026.681807:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:1.0:1788879026.681811:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879026.681815:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879026.681818:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879026.681819:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788879026.681829:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1788879026.681831:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879026.681835:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681837:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.681841:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:1.0:1788879026.681843:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788879026.681851:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879026.681852:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879026.681862:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.681866:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304638505 00000100:00000001:1.0:1788879026.681867:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879026.681874:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879026.681879:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681881:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.681884:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1788879026.681889:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879026.681894:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681897:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879026.681899:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879026.681964:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.681976:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.681980:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304638505 00000100:00100000:1.0:1788879026.681991:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100aa00 x1875765290122496/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879026.682001:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788879026.682015:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879026.682020:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879026.682024:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.682031:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.682035:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1788879026.682041:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879026.682055:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879026/real 1788879026] req@ffff94e51100a680 x1875765290122112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788879026.682080:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879026.682084:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879026.682091:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879026.682096:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879026.682099:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682104:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879026.682107:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879026.682115:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100a680 x1875765290122112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879026.682132:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682138:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879026.682144:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682147:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879026.682149:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682153:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879026.682157:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879026.682163:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879026.682170:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879026.682175:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879026.682183:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290122112:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879026.682192:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682196:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879026.682200:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879026.682204:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879026.682210:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879026.682214:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879026.682218:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879026.682224:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879026.682226:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879026.682230:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879026.682232:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682236:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682244:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879026.682247:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.682249:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879026.682252:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879026.682254:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879026.682256:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879026.682258:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879026.682261:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879026.682263:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879026.682266:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879026.682271:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290122240:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788879026.682275:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879026.682278:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879026.682281:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879026.682283:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879026.682287:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.682297:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879026.682299:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:0.0:1788879026.682305:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788879026.682325:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879026.682332:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879026.682335:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879026.682337:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.682339:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879026.682342:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.682348:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304637964 00000100:00000001:0.0:1788879026.682351:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304638505 00000100:00000001:0.0:1788879026.682355:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304637964 00000100:00000001:0.0:1788879026.682358:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304638505 00000100:00000001:0.0:1788879026.682360:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879026.682363:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.682365:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788879026.682371:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879026/real 1788879026] req@ffff94e51100b800 x1875765290122240/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788879026.682381:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879026.682382:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879026.682385:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879026.682387:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879026.682389:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682402:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879026.682404:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879026.682409:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290122240/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879026.682417:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682420:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879026.682424:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682426:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879026.682428:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682430:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879026.682444:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879026.682447:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879026.682452:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879026.682455:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879026.682459:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290122240:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788879026.682463:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879026.682464:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879026.682466:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879026.682468:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879026.682470:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879026.682473:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879026.682476:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290122368:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879026.682481:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879026.682483:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879026.682485:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879026.682487:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879026.682490:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.682495:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879026.682497:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:0.0:1788879026.682501:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788879026.682512:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879026.682516:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879026.682519:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879026.682521:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.682523:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879026.682526:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.682530:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304637962 00000100:00000001:0.0:1788879026.682532:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304638505 00000100:00000001:0.0:1788879026.682536:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304637962 00000100:00000001:0.0:1788879026.682539:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304638505 00000100:00000001:0.0:1788879026.682541:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879026.682544:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.682546:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788879026.682552:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879026/real 1788879026] req@ffff94e51100b480 x1875765290122368/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788879026.682569:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879026.682571:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879026.682576:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879026.682579:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879026.682582:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682585:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879026.682588:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879026.682596:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290122368/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879026.682611:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682633:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879026.682640:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682643:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879026.682646:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682649:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879026.682668:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879026.682672:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879026.682677:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879026.682691:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879026.682726:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290122368:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879026.682733:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879026.682735:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879026.682737:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879026.682740:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879026.682743:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879026.682745:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879026.682749:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290122496:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879026.682754:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879026.682756:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879026.682759:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879026.682761:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879026.682765:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.682774:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879026.682775:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:0.0:1788879026.682780:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788879026.682798:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879026.682804:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879026.682807:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879026.682809:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.682811:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879026.682815:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.682819:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304637962 00000100:00000001:0.0:1788879026.682822:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304638505 00000100:00000001:0.0:1788879026.682825:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304637962 00000100:00000001:0.0:1788879026.682828:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304638505 00000100:00000001:0.0:1788879026.682831:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879026.682833:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.682835:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788879026.682840:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879026/real 1788879026] req@ffff94e51100aa00 x1875765290122496/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788879026.682849:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879026.682851:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879026.682853:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879026.682856:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879026.682857:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682860:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879026.682863:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879026.682867:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290122496/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879026.682876:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682879:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879026.682881:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682883:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879026.682885:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682887:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879026.682899:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879026.682932:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879026.682936:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879026.682939:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879026.682943:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290122496:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879026.682947:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682950:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879026.682951:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879026.682953:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879026.682957:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879026.682959:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879026.682960:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879026.682964:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879026.682965:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879026.682968:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879026.682970:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682972:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879026.682973:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879026.682975:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879026.682977:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879026.682979:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879026.682980:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879026.682982:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879026.682983:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879026.682985:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879026.682987:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.682989:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879026.682991:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879026.682992:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879026.682994:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879026.682996:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879026.682997:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879026.682999:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879026.683000:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879026.683002:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879026.683003:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.683005:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879026.683010:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879026.683012:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879026.683014:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879026.683016:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879026.745052:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879026.745061:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879026.745066:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879026.745074:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879026.745081:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879026.745086:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879026.745091:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879026.745095:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879026.745099:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879027.449074:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879027.449080:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879027.449085:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879027.449086:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879027.449158:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879027.449164:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879027.449169:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879027.769014:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879027.769023:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879027.769026:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879027.769032:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879027.769036:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879027.769042:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879027.769045:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879027.769048:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879027.769051:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879028.473089:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879028.473099:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879028.473107:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879028.473109:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879028.473241:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879028.473248:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879028.473254:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879028.793035:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879028.793047:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879028.793050:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879028.793058:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879028.793074:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879028.793080:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879028.793084:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879028.793088:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879028.793092:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879029.497052:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879029.497062:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879029.497070:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879029.497072:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879029.497101:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879029.497105:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879029.497110:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879029.817078:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879029.817090:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879029.817094:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879029.817103:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879029.817109:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879029.817115:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879029.817136:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879029.817141:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879029.817146:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879030.521049:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879030.521058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879030.521065:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879030.521067:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879030.521103:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879030.521106:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879030.521109:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879030.841023:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879030.841033:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879030.841036:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879030.841044:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879030.841050:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879030.841055:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879030.841059:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879030.841063:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879030.841067:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879031.545015:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879031.545025:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879031.545032:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879031.545034:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879031.545117:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879031.545125:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879031.545131:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879031.801208:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879031.801230:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879031.801236:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879031.801244:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879031.801254:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879031.801268:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9970 00000100:00080000:3.0:1788879031.801289:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879031.801295:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879031.801299:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879031.801331:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879031.801336:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879031.801341:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879031.801346:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879031.801351:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879031.801353:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879031.801360:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879031.801365:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879031.801381:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879031.801404:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879031.801409:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.801412:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879031.801416:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879031.801436:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879031.801439:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.801443:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879031.801448:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879031.801456:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.801475:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879031.801491:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290122624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879031.801506:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879031.801521:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.801561:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879031.801566:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879031.801572:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879031.801574:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879031.801577:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879031.801582:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879031.801587:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9970 00000100:00080000:3.0:1788879031.801593:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879031.801597:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879031.801616:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788879031.801620:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788879031.801623:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879031.801626:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1788879031.801630:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788879031.801630:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879031.801636:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788879031.801636:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879031.801640:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879031.801641:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879031.801643:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788879031.801645:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879031.801650:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:0.0:1788879031.801653:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879031.801654:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879031.801658:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879031.801659:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879031.801664:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879031.801691:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788879031.801696:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290122624:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788879031.801696:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.801716:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879031.801721:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1788879031.801723:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1788879031.801727:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:3.0:1788879031.801728:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879031.801731:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.801731:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879031.801735:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879031.801738:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879031.801740:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879031.801743:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.801743:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879031.801745:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879031.801748:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.801756:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879031.801764:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290122752/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879031.801786:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879031.801790:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:3.0:1788879031.801796:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879031.801798:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879031.801800:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.801805:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879031.801810:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879031.801816:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879031.801818:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879031.801821:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879031.801826:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879031.801830:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9970 00000100:00080000:3.0:1788879031.801836:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879031.801840:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879031.801843:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879031.801849:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879031.801851:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879031.801855:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879031.801859:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879031.801863:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879031.801866:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:0.0:1788879031.801867:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1788879031.801873:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879031.801877:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879031.801880:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879031.801885:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879031.801888:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.801891:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879031.801893:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00100000:3.0:1788879031.801895:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788879031.801900:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00000001:3.0:1788879031.801900:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.801903:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.801904:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.801909:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879031.801910:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879031.801912:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.801915:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879031.801918:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1788879031.801919:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879031.801922:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.801927:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304643625 00000100:00000001:3.0:1788879031.801930:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879031.802002:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290122880/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879031.802034:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879031.802038:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.802044:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802047:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788879031.802048:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879031.802053:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879031.802055:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788879031.802057:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304643626 00000100:00080000:3.0:1788879031.802058:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879031.802062:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:0.0:1788879031.802064:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788879031.802066:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9970 00000100:00000001:0.0:1788879031.802069:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879031.802073:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879031.802074:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.802076:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879031.802078:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879031.802078:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879031.802083:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879031.802085:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:0.0:1788879031.802086:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879031/real 1788879031] req@ffff94e50449f480 x1875765290122624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788879031.802089:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879031.802111:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879031.802115:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879031.802118:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879031.802125:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879031.802130:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879031.802133:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:0.0:1788879031.802139:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1788879031.802139:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:0.0:1788879031.802144:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788879031.802144:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.802147:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879031.802151:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000008:00100000:3.0:1788879031.802151:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788879031.802156:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000008:00000001:3.0:1788879031.802156:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.802159:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802160:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.802165:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788879031.802168:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788879031.802170:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879031.802170:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.802174:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879031.802177:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1788879031.802179:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290122624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879031.802181:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879031.802191:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802195:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879031.802199:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290123008/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879031.802202:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879031.802208:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.802212:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879031.802215:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879031.802215:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879031.802219:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802220:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879031.802223:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802224:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879031.802229:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879031.802237:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879031.802242:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879031.802249:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290122624:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879031.802259:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.802263:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879031.802266:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879031.802271:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879031.802278:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879031.802283:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879031.802287:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879031.802295:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879031.802298:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879031.802303:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879031.802306:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.802311:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.802326:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879031.802330:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802333:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879031.802336:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879031.802338:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879031.802357:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879031.802359:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879031.802362:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879031.802364:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879031.802366:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879031.802370:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290122752:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788879031.802374:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879031.802376:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879031.802378:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879031.802380:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879031.802383:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802392:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879031.802393:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:0.0:1788879031.802399:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788879031.802435:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879031.802442:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879031.802445:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879031.802447:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802450:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879031.802453:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802460:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304643084 00000100:00000001:0.0:1788879031.802464:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304643625 00000100:00000001:0.0:1788879031.802467:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304643084 00000100:00000001:0.0:1788879031.802470:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304643626 00000100:00000001:0.0:1788879031.802473:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879031.802475:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802478:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788879031.802483:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879031/real 1788879031] req@ffff94e50449c700 x1875765290122752/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788879031.802503:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879031.802505:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879031.802508:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879031.802510:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879031.802512:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.802514:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879031.802517:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879031.802539:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290122752/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879031.802546:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.802548:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879031.802550:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.802552:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879031.802553:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.802554:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879031.802573:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879031.802575:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879031.802578:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879031.802581:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879031.802584:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290122752:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788879031.802588:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879031.802590:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879031.802591:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879031.802593:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879031.802612:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879031.802614:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879031.802618:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290122880:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879031.802622:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879031.802624:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879031.802627:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879031.802629:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879031.802633:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802640:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879031.802642:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:0.0:1788879031.802648:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788879031.802661:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879031.802666:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879031.802693:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879031.802696:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802698:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879031.802719:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802723:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304643082 00000100:00000001:0.0:1788879031.802726:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304643625 00000100:00000001:0.0:1788879031.802728:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304643082 00000100:00000001:0.0:1788879031.802731:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304643626 00000100:00000001:0.0:1788879031.802733:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879031.802735:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802736:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788879031.802741:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879031/real 1788879031] req@ffff94e50449e300 x1875765290122880/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788879031.802748:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879031.802749:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879031.802751:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879031.802753:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879031.802754:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.802756:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879031.802758:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879031.802761:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290122880/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879031.802785:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.802788:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879031.802792:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.802794:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879031.802796:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.802798:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879031.802817:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879031.802820:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879031.802824:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879031.802827:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879031.802831:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290122880:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879031.802835:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879031.802837:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879031.802838:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879031.802841:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879031.802843:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879031.802846:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879031.802850:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290123008:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879031.802854:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879031.802856:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879031.802858:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879031.802860:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879031.802865:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802871:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879031.802872:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:0.0:1788879031.802877:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788879031.802892:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879031.802897:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879031.802899:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879031.802901:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802903:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879031.802906:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802910:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304643082 00000100:00000001:0.0:1788879031.802913:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304643625 00000100:00000001:0.0:1788879031.802917:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304643082 00000100:00000001:0.0:1788879031.802920:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304643626 00000100:00000001:0.0:1788879031.802923:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879031.802925:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.802927:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788879031.802962:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879031/real 1788879031] req@ffff94e50449f100 x1875765290123008/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788879031.802973:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879031.802975:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879031.802979:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879031.802981:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879031.802983:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.802985:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879031.802987:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879031.802992:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290123008/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879031.803002:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.803004:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879031.803008:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.803009:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879031.803011:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.803013:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879031.803047:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879031.803049:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879031.803052:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879031.803054:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879031.803057:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290123008:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879031.803061:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.803063:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879031.803065:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879031.803067:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879031.803069:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879031.803071:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879031.803072:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879031.803075:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879031.803076:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879031.803078:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879031.803080:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.803082:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879031.803083:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879031.803084:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879031.803087:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879031.803088:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879031.803106:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879031.803108:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879031.803109:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879031.803112:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879031.803113:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.803115:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879031.803116:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879031.803118:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879031.803119:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879031.803121:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879031.803122:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879031.803124:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879031.803125:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879031.803127:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879031.803128:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.803130:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879031.803134:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879031.803136:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879031.803138:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879031.803141:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879031.865047:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879031.865057:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879031.865060:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879031.865067:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879031.865072:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879031.865077:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879031.865081:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879031.865084:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879031.865087:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879032.569073:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879032.569087:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879032.569112:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879032.569116:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879032.569120:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879032.569123:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879032.569127:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879032.569129:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879032.569132:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879032.569134:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879032.569208:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879032.569213:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879032.569217:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879032.889038:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879032.889049:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879032.889052:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879032.889060:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879032.889065:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879032.889070:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879032.889073:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879032.889076:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879032.889080:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879033.593068:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879033.593076:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879033.593082:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879033.593084:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879033.593125:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879033.593129:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879033.593132:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879033.913029:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879033.913037:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879033.913040:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879033.913046:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879033.913050:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879033.913054:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879033.913056:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879033.913059:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879033.913062:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879034.616985:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879034.616992:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879034.616998:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879034.616999:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879034.617032:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879034.617034:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879034.617037:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879034.937032:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879034.937040:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879034.937042:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879034.937048:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879034.937053:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879034.937057:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879034.937060:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879034.937063:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879034.937067:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879035.641042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879035.641054:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879035.641073:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879035.641077:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879035.641083:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879035.641087:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879035.641089:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879035.641093:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879035.641098:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879035.641100:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879035.641181:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879035.641189:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879035.641197:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879035.961033:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879035.961044:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879035.961049:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879035.961074:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879035.961080:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879035.961085:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879035.961089:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879035.961092:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879035.961096:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879036.665079:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879036.665092:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879036.665104:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879036.665108:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879036.665163:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879036.665170:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879036.665184:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879036.921157:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879036.921191:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879036.921195:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879036.921201:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879036.921209:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879036.921222:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9975 00000100:00080000:1.0:1788879036.921245:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879036.921250:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879036.921253:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879036.921267:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879036.921271:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879036.921277:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879036.921281:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879036.921284:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879036.921287:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879036.921294:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879036.921297:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879036.921302:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879036.921308:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879036.921313:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879036.921316:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879036.921320:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879036.921339:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879036.921342:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879036.921347:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879036.921351:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879036.921359:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879036.921381:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879036.921395:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100bb80 x1875765290123136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879036.921410:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879036.921417:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879036.921440:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879036.921444:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879036.921449:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879036.921451:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879036.921454:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879036.921458:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879036.921461:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9975 00000100:00080000:1.0:1788879036.921466:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879036.921470:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879036.921473:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879036.921477:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879036.921479:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879036.921482:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879036.921486:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879036.921490:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879036.921492:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879036.921497:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879036.921500:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879036.921502:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788879036.921507:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1788879036.921512:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879036.921516:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1788879036.921518:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.921523:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879036.921525:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879036.921532:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:1.0:1788879036.921536:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1788879036.921538:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1788879036.921545:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879036.921549:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788879036.921552:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879036.921557:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879036.921561:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879036.921565:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879036.921566:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879036.921575:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879036.921579:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1788879036.921582:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290123136:192.168.203.129@tcp:250:kworker.0 02000000:00000001:1.0:1788879036.921584:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.921598:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788879036.921600:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879036.921607:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:1.0:1788879036.921615:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008700 x1875765290123264/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1788879036.921616:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879036.921626:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879036.921661:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879036.921662:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879036.921670:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879036.921676:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879036.921680:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879036.921686:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879036.921688:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879036.921691:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879036.921695:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879036.921699:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9975 00000100:00000001:0.0:1788879036.921704:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1788879036.921705:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879036.921715:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286523264 : -117763423028352 : ffff94e51100bb80) 00000100:00000001:1.0:1788879036.921715:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879036.921719:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879036.921730:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1788879036.921733:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1788879036.921733:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879036.921741:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879036.921746:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879036.921748:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879036.921751:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879036.921755:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879036.921759:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879036.921762:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879036.921766:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879036.921769:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879036.921772:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879036.921775:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879036.921778:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879036.921781:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879036.921788:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879036.921790:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879036.921794:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879036.921796:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879036.921798:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879036.921808:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1788879036.921819:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:1.0:1788879036.921820:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008e00 x1875765290123392/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879036.921836:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879036.921839:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879036.921844:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.921845:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:1.0:1788879036.921848:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879036.921855:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879036.921856:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879036.921862:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1788879036.921864:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879036.921867:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879036.921870:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879036.921874:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879036.921882:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9975 00000100:00000001:0.0:1788879036.921883:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879036.921894:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879036.921896:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304648745 00000100:00000001:1.0:1788879036.921963:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879036.921972:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879036.921979:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879036.921982:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879036.921985:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879036.921987:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:1.0:1788879036.921992:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879036.921999:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879036.922002:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:0.0:1788879036.922004:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304648746 00000100:00000001:0.0:1788879036.922018:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1788879036.922019:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879036.922027:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879036.922029:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922035:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879036.922040:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1788879036.922043:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1788879036.922053:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788879036.922057:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879036/real 1788879036] req@ffff94e51100bb80 x1875765290123136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788879036.922065:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879036.922077:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879036.922087:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000400:00000001:0.0:1788879036.922089:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879036.922096:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00000001:1.0:1788879036.922099:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922104:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879036.922108:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922111:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879036.922117:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.922121:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1788879036.922124:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1788879036.922125:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788879036.922135:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879036.922137:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100bb80 x1875765290123136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879036.922146:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879036.922153:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879036.922159:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879036.922177:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922178:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.922192:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:1.0:1788879036.922199:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008000 x1875765290123520/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879036.922201:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.922208:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879036.922213:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.922221:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879036.922226:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879036.922230:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00080000:0.0:1788879036.922233:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1788879036.922238:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879036.922247:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879036.922248:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922255:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879036.922281:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290123136:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879036.922290:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.922294:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879036.922298:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879036.922302:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879036.922307:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879036.922311:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879036.922314:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879036.922320:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879036.922323:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879036.922327:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879036.922329:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.922331:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.922342:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879036.922345:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922348:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879036.922350:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879036.922352:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879036.922354:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879036.922356:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879036.922359:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879036.922362:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879036.922364:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879036.922369:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290123264:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788879036.922374:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879036.922377:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879036.922379:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879036.922382:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879036.922386:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922394:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879036.922396:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286509824 : -117763423041792 : ffff94e511008700) 00000100:00000001:0.0:1788879036.922403:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788879036.922421:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879036.922427:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879036.922430:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879036.922432:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922434:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879036.922437:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922443:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304648204 00000100:00000001:0.0:1788879036.922447:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304648745 00000100:00000001:0.0:1788879036.922450:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304648204 00000100:00000001:0.0:1788879036.922454:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304648746 00000100:00000001:0.0:1788879036.922456:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879036.922459:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922461:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788879036.922466:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879036/real 1788879036] req@ffff94e511008700 x1875765290123264/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788879036.922476:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879036.922478:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879036.922481:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879036.922483:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879036.922485:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.922488:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879036.922490:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879036.922494:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008700 x1875765290123264/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879036.922512:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.922515:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879036.922519:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.922520:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879036.922522:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.922524:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879036.922546:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879036.922549:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879036.922553:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879036.922555:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879036.922559:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290123264:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788879036.922565:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879036.922566:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879036.922568:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879036.922571:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879036.922573:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879036.922576:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879036.922579:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290123392:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879036.922583:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879036.922585:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879036.922586:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879036.922588:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879036.922592:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922599:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879036.922601:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286511616 : -117763423040000 : ffff94e511008e00) 00000100:00000001:0.0:1788879036.922606:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788879036.922617:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879036.922622:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879036.922624:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879036.922626:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922630:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879036.922667:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922675:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304648202 00000100:00000001:0.0:1788879036.922681:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304648745 00000100:00000001:0.0:1788879036.922685:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304648202 00000100:00000001:0.0:1788879036.922690:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304648746 00000100:00000001:0.0:1788879036.922694:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879036.922699:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922703:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788879036.922713:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879036/real 1788879036] req@ffff94e511008e00 x1875765290123392/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788879036.922730:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879036.922733:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879036.922739:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879036.922743:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879036.922746:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.922750:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879036.922753:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879036.922760:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008e00 x1875765290123392/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879036.922774:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.922779:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879036.922784:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.922787:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879036.922790:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.922794:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879036.922818:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879036.922823:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879036.922828:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879036.922831:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879036.922836:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290123392:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879036.922841:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879036.922843:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879036.922845:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879036.922848:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879036.922851:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879036.922854:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879036.922858:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290123520:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879036.922862:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879036.922865:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879036.922867:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879036.922869:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879036.922874:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922883:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879036.922884:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508032 : -117763423043584 : ffff94e511008000) 00000100:00000001:0.0:1788879036.922890:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1788879036.922940:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879036.922947:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879036.922950:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879036.922952:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922955:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879036.922958:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922963:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304648203 00000100:00000001:0.0:1788879036.922967:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304648745 00000100:00000001:0.0:1788879036.922969:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304648203 00000100:00000001:0.0:1788879036.922973:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304648746 00000100:00000001:0.0:1788879036.922975:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879036.922978:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.922980:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1788879036.922985:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879036/real 1788879036] req@ffff94e511008000 x1875765290123520/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1788879036.923003:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1788879036.923005:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879036.923008:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879036.923011:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879036.923012:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.923015:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879036.923017:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879036.923022:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008000 x1875765290123520/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879036.923030:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.923033:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879036.923036:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.923038:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879036.923039:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.923042:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879036.923060:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879036.923063:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879036.923066:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879036.923068:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879036.923072:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290123520:192.168.203.129@tcp:8:kworker.0 00000100:00000001:0.0:1788879036.923076:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.923078:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879036.923080:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879036.923082:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879036.923085:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879036.923087:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879036.923088:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879036.923092:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879036.923093:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879036.923096:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879036.923098:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.923101:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879036.923103:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879036.923104:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879036.923107:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879036.923109:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879036.923110:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879036.923112:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879036.923114:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879036.923116:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879036.923117:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.923119:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879036.923120:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879036.923122:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879036.923123:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879036.923125:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879036.923126:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879036.923128:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879036.923129:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879036.923131:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879036.923132:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.923134:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879036.923139:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879036.923141:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879036.923143:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879036.923145:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879036.985067:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879036.985077:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879036.985079:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879036.985088:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879036.985092:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879036.985097:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879036.985103:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879036.985108:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879036.985113:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879037.689085:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879037.689094:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879037.689102:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879037.689105:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879037.689134:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879037.689138:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879037.689143:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879038.009058:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879038.009069:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879038.009073:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879038.009081:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879038.009086:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879038.009092:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879038.009096:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879038.009101:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879038.009105:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879038.713062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879038.713073:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879038.713083:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879038.713086:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879038.713131:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879038.713136:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879038.713141:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879039.032996:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879039.033004:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879039.033006:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879039.033011:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879039.033015:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879039.033019:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879039.033022:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879039.033025:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879039.033028:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879039.737043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879039.737051:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879039.737057:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879039.737059:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879039.737178:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879039.737186:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879039.737193:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879040.057015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879040.057027:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879040.057030:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879040.057038:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879040.057043:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879040.057049:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879040.057053:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879040.057057:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879040.057061:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879040.761011:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879040.761022:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879040.761032:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879040.761035:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879040.761134:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879040.761139:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879040.761144:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879041.081007:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879041.081017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879041.081020:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879041.081028:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879041.081033:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879041.081037:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879041.081041:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879041.081044:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879041.081047:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879041.785039:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879041.785047:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879041.785069:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879041.785071:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879041.785097:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879041.785100:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879041.785104:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879042.041102:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879042.041114:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879042.041117:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879042.041122:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879042.041128:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879042.041136:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9980 00000100:00080000:0.0:1788879042.041149:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879042.041153:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879042.041156:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879042.041166:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879042.041169:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879042.041174:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879042.041177:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879042.041180:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879042.041182:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879042.041187:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879042.041192:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879042.041196:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879042.041202:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879042.041209:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879042.041214:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879042.041218:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879042.041235:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879042.041239:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879042.041243:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879042.041247:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879042.041254:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879042.041272:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879042.041289:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290123648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879042.041310:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879042.041318:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879042.041348:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879042.041353:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879042.041358:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879042.041360:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879042.041362:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879042.041366:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879042.041371:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9980 00000100:00080000:0.0:1788879042.041377:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879042.041381:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879042.041383:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879042.041389:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879042.041391:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879042.041394:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879042.041399:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788879042.041400:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879042.041407:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879042.041408:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879042.041410:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879042.041415:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1788879042.041421:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879042.041423:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879042.041432:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879042.041433:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879042.041443:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879042.041445:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879042.041451:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879042.041452:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879042.041459:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879042.041461:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879042.041468:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879042.041474:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879042.041478:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879042.041481:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879042.041487:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290123648:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788879042.041491:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879042.041496:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.041499:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788879042.041501:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879042.041506:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879042.041508:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879042.041510:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879042.041513:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879042.041519:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879042.041522:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879042.041527:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879042.041536:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290123776/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879042.041548:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879042.041552:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.041553:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879042.041559:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.041560:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00080000:0.0:1788879042.041571:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879042.041572:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788879042.041580:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879042.041584:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879042.041586:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879042.041590:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879042.041594:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9980 00000100:00080000:0.0:1788879042.041640:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879042.041644:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879042.041646:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:1.0:1788879042.041654:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:0.0:1788879042.041664:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879042.041668:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879042.041672:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879042.041673:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:0.0:1788879042.041681:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879042.041683:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879042.041687:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879042.041689:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879042.041694:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879042.041696:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:0.0:1788879042.041707:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879042.041707:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879042.041718:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.041720:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304653865 02000000:00000001:0.0:1788879042.041724:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879042.041731:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879042.041735:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.041737:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788879042.041739:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879042.041747:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879042.041751:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304653865 00000008:00000001:0.0:1788879042.041753:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879042.041762:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.041764:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879042.041770:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879042.041771:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879042.041776:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879042.041777:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.041784:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879042.041785:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879042.041791:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879042.041795:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879042.041799:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879042/real 1788879042] req@ffff94e50897a300 x1875765290123648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879042.041804:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879042.041821:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978700 x1875765290123904/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1788879042.041824:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879042.041829:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879042.041835:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879042.041841:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879042.041842:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879042.041845:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879042.041850:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.041853:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879042.041859:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879042.041860:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879042.041870:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788879042.041877:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290123648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879042.041879:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879042.041885:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879042.041890:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879042.041901:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879042.041941:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.041946:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879042.041950:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.041952:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879042.041954:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879042.041962:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9980 00000100:00000001:1.0:1788879042.041962:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879042.041968:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879042.041972:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1788879042.041975:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1788879042.041981:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879042.041988:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879042.041988:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879042.041999:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:0.0:1788879042.042000:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879042.042007:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:1.0:1788879042.042008:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290123648:192.168.203.129@tcp:250:kworker.0 00000020:00000001:0.0:1788879042.042011:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879042.042020:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879042.042023:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879042.042024:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879042.042030:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879042.042032:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879042.042037:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00080000:0.0:1788879042.042043:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788879042.042045:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1788879042.042054:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879042.042056:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:0.0:1788879042.042057:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:1.0:1788879042.042062:0:2361:0:(genops.c:1216:class_import_put()) Process entered 02000000:00000001:0.0:1788879042.042065:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1788879042.042067:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879042.042074:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879042.042077:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1788879042.042078:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879042.042082:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000008:00100000:0.0:1788879042.042085:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879042.042089:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000008:00000001:0.0:1788879042.042091:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042094:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879042.042097:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042101:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879042.042107:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879042.042111:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042114:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879042.042115:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879042.042121:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879042.042122:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879042.042129:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788879042.042131:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042135:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879042.042139:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879042.042141:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042142:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879042.042147:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879042.042153:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:0.0:1788879042.042156:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897aa00 x1875765290124032/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879042.042157:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879042.042167:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879042.042180:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290123776:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788879042.042183:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879042.042190:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788879042.042192:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879042.042196:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879042.042201:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879042.042206:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879042.042206:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879042.042213:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042224:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879042.042227:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:1.0:1788879042.042233:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879042.042252:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879042.042259:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879042.042263:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879042.042265:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042267:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879042.042270:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042275:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304653324 00000100:00000001:1.0:1788879042.042279:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304653865 00000100:00000001:1.0:1788879042.042282:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304653324 00000100:00000001:1.0:1788879042.042285:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304653865 00000100:00000001:1.0:1788879042.042288:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879042.042290:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042292:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879042.042297:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879042/real 1788879042] req@ffff94e508978a80 x1875765290123776/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879042.042309:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879042.042311:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879042.042314:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879042.042316:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879042.042318:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042320:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879042.042322:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879042.042327:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290123776/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879042.042336:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042338:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879042.042341:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042343:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879042.042344:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042346:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879042.042363:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879042.042366:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879042.042369:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879042.042372:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879042.042376:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290123776:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879042.042380:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879042.042382:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879042.042384:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879042.042387:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879042.042389:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879042.042391:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879042.042395:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290123904:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879042.042399:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879042.042401:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879042.042403:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879042.042405:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879042.042409:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042414:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879042.042416:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00000001:1.0:1788879042.042420:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879042.042432:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879042.042437:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879042.042439:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879042.042441:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042443:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879042.042445:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042449:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304653322 00000100:00000001:1.0:1788879042.042452:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304653865 00000100:00000001:1.0:1788879042.042455:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304653322 00000100:00000001:1.0:1788879042.042458:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304653865 00000100:00000001:1.0:1788879042.042461:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879042.042463:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042465:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879042.042470:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879042/real 1788879042] req@ffff94e508978700 x1875765290123904/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879042.042479:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879042.042480:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879042.042483:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879042.042485:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879042.042486:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042488:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879042.042490:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879042.042494:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290123904/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879042.042503:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042505:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879042.042509:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042511:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879042.042513:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042515:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879042.042529:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879042.042533:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879042.042537:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879042.042540:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879042.042543:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290123904:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879042.042547:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042550:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879042.042551:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879042.042553:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879042.042555:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879042.042557:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879042.042558:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879042.042561:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879042.042562:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879042.042565:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879042.042566:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042568:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879042.042570:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879042.042571:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879042.042573:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879042.042575:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879042.042576:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879042.042578:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879042.042580:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879042.042582:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879042.042583:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042585:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042589:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879042.042590:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042593:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879042.042594:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879042.042596:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879042.042618:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879042.042621:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879042.042623:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879042.042626:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879042.042628:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879042.042632:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290124032:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879042.042636:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879042.042638:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879042.042640:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879042.042643:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879042.042647:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042653:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879042.042655:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:1.0:1788879042.042659:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879042.042672:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879042.042678:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879042.042680:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879042.042682:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042684:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879042.042687:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042691:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304653322 00000100:00000001:1.0:1788879042.042695:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304653865 00000100:00000001:1.0:1788879042.042698:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304653322 00000100:00000001:1.0:1788879042.042700:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304653865 00000100:00000001:1.0:1788879042.042703:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879042.042706:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042708:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879042.042713:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879042/real 1788879042] req@ffff94e50897aa00 x1875765290124032/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879042.042722:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879042.042723:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879042.042726:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879042.042728:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879042.042730:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042733:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879042.042735:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879042.042739:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290124032/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879042.042748:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042751:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879042.042753:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042755:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879042.042757:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042759:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879042.042773:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879042.042776:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879042.042779:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879042.042781:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879042.042785:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290124032:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879042.042789:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042791:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879042.042793:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879042.042795:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879042.042797:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879042.042798:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879042.042800:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879042.042803:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879042.042804:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879042.042807:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879042.042808:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042811:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.042814:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879042.042816:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.042819:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879042.042821:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879042.105024:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879042.105032:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879042.105035:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879042.105043:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879042.105047:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879042.105051:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.105055:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879042.105058:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879042.105061:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879042.809096:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879042.809104:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879042.809110:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879042.809115:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879042.809118:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879042.809122:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879042.809126:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879042.809129:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879042.809132:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879042.809135:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879042.809161:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879042.809164:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879042.809167:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879043.129057:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879043.129069:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879043.129073:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879043.129081:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879043.129087:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879043.129092:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879043.129096:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879043.129100:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879043.129104:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879043.833112:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879043.833126:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879043.833138:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879043.833142:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879043.833205:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879043.833213:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879043.833220:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879044.153074:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879044.153085:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879044.153088:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879044.153097:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879044.153103:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879044.153109:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879044.153113:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879044.153117:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879044.153121:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879044.857047:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879044.857055:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879044.857061:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879044.857063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879044.857176:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879044.857182:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879044.857186:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879045.177069:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879045.177078:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879045.177080:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879045.177086:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879045.177091:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879045.177095:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879045.177098:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879045.177101:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879045.177104:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879045.881025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879045.881047:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879045.881051:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879045.881056:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879045.881062:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879045.881065:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879045.881068:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879045.881070:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879045.881075:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879045.881077:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879045.881100:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879045.881103:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879045.881107:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879046.201023:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879046.201032:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879046.201035:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879046.201042:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879046.201048:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879046.201053:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879046.201057:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879046.201061:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879046.201065:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879046.905035:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879046.905043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879046.905049:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879046.905051:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879046.905073:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879046.905077:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879046.905080:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879047.161104:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879047.161116:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879047.161121:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879047.161126:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879047.161132:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879047.161142:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9986 00000100:00080000:0.0:1788879047.161156:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879047.161160:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879047.161162:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879047.161172:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879047.161175:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879047.161180:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879047.161183:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879047.161186:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879047.161188:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879047.161193:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879047.161196:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879047.161200:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879047.161204:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879047.161208:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879047.161211:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879047.161214:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879047.161228:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879047.161230:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879047.161233:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879047.161237:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879047.161242:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879047.161258:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879047.161270:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897aa00 x1875765290124160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879047.161283:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879047.161289:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879047.161309:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879047.161314:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879047.161319:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879047.161321:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879047.161323:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879047.161328:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879047.161331:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9986 00000100:00080000:0.0:1788879047.161335:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879047.161338:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879047.161340:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879047.161344:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879047.161346:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879047.161349:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879047.161351:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879047.161354:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879047.161356:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879047.161362:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879047.161365:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879047.161368:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879047.161375:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879047.161379:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879047.161381:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879047.161385:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:0.0:1788879047.161386:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879047.161396:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.161396:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879047.161403:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879047.161404:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.161410:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788879047.161414:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879047.161416:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879047.161419:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879047.161425:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788879047.161427:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879047.161433:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879047.161434:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879047.161442:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879047.161444:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879047.161454:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290124160:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879047.161461:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.161463:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879047.161472:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879047.161476:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879047.161482:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1788879047.161483:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978700 x1875765290124288/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879047.161489:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879047.161509:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879047.161515:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879047.161516:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.161524:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:0.0:1788879047.161529:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879047.161538:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879047.161538:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788879047.161548:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879047.161550:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879047.161553:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879047.161557:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879047.161588:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9986 00000100:00080000:0.0:1788879047.161621:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879047.161626:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879047.161630:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:1.0:1788879047.161641:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:0.0:1788879047.161642:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879047.161649:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879047.161654:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879047.161660:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879047.161674:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879047.161676:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879047.161679:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879047.161685:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879047.161690:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879047.161695:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879047.161695:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879047.161707:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879047.161708:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879047.161717:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879047.161720:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304658985 02000000:00000001:0.0:1788879047.161722:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879047.161729:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879047.161732:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879047.161736:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879047.161737:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00000001:0.0:1788879047.161743:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879047.161748:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.161750:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304658985 02000000:00000001:0.0:1788879047.161757:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879047.161761:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879047.161766:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879047.161769:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879047.161774:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.161776:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879047.161781:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879047.161786:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1788879047.161791:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879047/real 1788879047] req@ffff94e50897aa00 x1875765290124160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788879047.161792:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879047.161808:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879047.161814:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879047.161819:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:0.0:1788879047.161824:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290124416/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879047.161827:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879047.161833:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879047.161838:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.161847:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879047.161852:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879047.161868:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290124160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879047.161869:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879047.161878:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879047.161888:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879047.161898:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879047.161900:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879047.161949:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879047.161951:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879047.161956:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879047.161959:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879047.161962:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879047.161964:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879047.161971:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879047.161983:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879047.161984:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879047.161991:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879047.162000:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9986 00000100:00080000:1.0:1788879047.162000:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879047.162015:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00080000:0.0:1788879047.162017:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879047.162021:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1788879047.162026:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879047.162036:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:1.0:1788879047.162037:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290124160:192.168.203.129@tcp:250:kworker.0 00000020:00000001:0.0:1788879047.162049:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879047.162055:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788879047.162056:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879047.162064:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879047.162067:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879047.162074:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788879047.162076:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788879047.162081:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1788879047.162086:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879047.162092:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879047.162092:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879047.162100:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00080000:0.0:1788879047.162106:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000020:00000001:1.0:1788879047.162109:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:0.0:1788879047.162112:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879047.162117:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879047.162121:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879047.162122:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879047.162128:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:0.0:1788879047.162130:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879047.162135:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879047.162138:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000008:00100000:0.0:1788879047.162139:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879047.162145:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788879047.162147:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.162154:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879047.162155:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879047.162168:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879047.162170:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.162171:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879047.162176:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879047.162177:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879047.162182:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879047.162185:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788879047.162190:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.162191:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879047.162196:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879047.162200:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879047.162202:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879047.162204:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879047.162207:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879047.162214:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788879047.162222:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290124544/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879047.162222:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879047.162236:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290124288:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788879047.162253:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879047.162253:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788879047.162262:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879047.162262:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879047.162265:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879047.162271:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879047.162272:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879047.162279:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.162292:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879047.162295:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00000001:1.0:1788879047.162301:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879047.162323:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879047.162329:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879047.162332:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879047.162334:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.162337:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879047.162340:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.162345:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304658444 00000100:00000001:1.0:1788879047.162349:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304658985 00000100:00000001:1.0:1788879047.162352:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304658444 00000100:00000001:1.0:1788879047.162355:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304658985 00000100:00000001:1.0:1788879047.162358:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879047.162360:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.162362:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879047.162368:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879047/real 1788879047] req@ffff94e508978700 x1875765290124288/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879047.162380:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879047.162382:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879047.162385:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879047.162387:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879047.162389:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162391:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879047.162394:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879047.162399:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290124288/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879047.162407:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162410:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879047.162413:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162415:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879047.162416:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162418:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879047.162432:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879047.162435:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879047.162439:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879047.162442:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879047.162446:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290124288:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879047.162450:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879047.162452:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879047.162454:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879047.162456:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879047.162459:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879047.162461:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879047.162465:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290124416:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879047.162468:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879047.162470:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879047.162473:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879047.162475:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879047.162478:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.162484:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879047.162485:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:1.0:1788879047.162490:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879047.162502:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879047.162506:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879047.162509:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879047.162511:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.162514:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879047.162519:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.162525:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304658442 00000100:00000001:1.0:1788879047.162530:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304658985 00000100:00000001:1.0:1788879047.162534:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304658442 00000100:00000001:1.0:1788879047.162539:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304658985 00000100:00000001:1.0:1788879047.162543:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879047.162546:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.162550:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879047.162558:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879047/real 1788879047] req@ffff94e508978a80 x1875765290124416/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879047.162605:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879047.162607:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879047.162614:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879047.162617:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879047.162620:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162624:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879047.162626:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879047.162635:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290124416/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879047.162646:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162649:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879047.162653:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162665:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879047.162667:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162670:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879047.162683:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879047.162686:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879047.162690:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879047.162693:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879047.162697:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290124416:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879047.162701:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162704:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879047.162706:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879047.162708:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879047.162711:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879047.162713:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879047.162715:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879047.162718:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879047.162720:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879047.162723:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879047.162725:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162727:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879047.162728:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879047.162730:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879047.162732:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879047.162733:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879047.162734:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879047.162736:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879047.162738:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879047.162739:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879047.162741:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162743:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162747:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879047.162749:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.162751:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879047.162753:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879047.162755:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879047.162757:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879047.162759:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879047.162762:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879047.162764:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879047.162767:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879047.162770:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290124544:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879047.162775:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879047.162777:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879047.162779:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879047.162782:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879047.162786:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.162793:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879047.162795:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:1.0:1788879047.162800:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879047.162817:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879047.162823:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879047.162825:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879047.162827:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.162829:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879047.162833:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.162837:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304658442 00000100:00000001:1.0:1788879047.162840:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304658985 00000100:00000001:1.0:1788879047.162843:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304658442 00000100:00000001:1.0:1788879047.162846:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304658985 00000100:00000001:1.0:1788879047.162849:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879047.162851:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.162853:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879047.162858:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879047/real 1788879047] req@ffff94e50897a300 x1875765290124544/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879047.162867:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879047.162868:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879047.162871:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879047.162873:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879047.162875:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162877:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879047.162880:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879047.162884:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290124544/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879047.162893:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162895:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879047.162898:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162900:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879047.162932:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162934:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879047.162947:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879047.162950:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879047.162954:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879047.162957:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879047.162961:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290124544:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879047.162966:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.162968:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879047.162970:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879047.162972:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879047.162975:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879047.162976:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879047.162978:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879047.162981:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879047.162983:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879047.162985:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879047.162998:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.163000:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.163004:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879047.163006:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.163009:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879047.163011:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879047.225057:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879047.225069:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879047.225074:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879047.225084:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879047.225091:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879047.225097:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.225102:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879047.225107:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879047.225112:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879047.929048:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879047.929056:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879047.929063:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879047.929065:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879047.929156:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879047.929162:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879047.929167:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879048.249022:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879048.249031:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879048.249034:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879048.249041:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879048.249046:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879048.249050:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879048.249054:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879048.249057:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879048.249060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879048.953038:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879048.953046:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879048.953053:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879048.953055:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879048.953158:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879048.953164:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879048.953170:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879049.273041:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879049.273052:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879049.273056:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879049.273064:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879049.273071:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879049.273076:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879049.273080:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879049.273083:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879049.273086:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879049.977056:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879049.977064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879049.977071:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879049.977073:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879049.977096:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879049.977100:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879049.977103:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879050.296992:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879050.297000:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879050.297003:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879050.297009:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879050.297014:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879050.297019:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879050.297022:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879050.297025:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879050.297029:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879051.001021:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879051.001033:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879051.001043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879051.001045:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879051.001077:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879051.001082:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879051.001088:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879051.321071:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879051.321082:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879051.321086:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879051.321094:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879051.321113:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879051.321119:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879051.321123:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879051.321127:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879051.321131:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879052.024981:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879052.024985:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879052.024989:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879052.024990:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879052.025066:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879052.025069:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879052.025072:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879052.280977:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879052.280985:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879052.280988:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879052.280991:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879052.280994:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879052.281000:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9991 00000100:00080000:0.0:1788879052.281012:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879052.281015:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879052.281016:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879052.281023:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879052.281026:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879052.281029:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879052.281031:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879052.281032:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879052.281034:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879052.281037:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879052.281039:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879052.281041:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879052.281043:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879052.281045:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879052.281047:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879052.281049:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879052.281057:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879052.281058:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879052.281061:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879052.281064:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879052.281067:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879052.281077:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879052.281085:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290124672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879052.281093:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879052.281097:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879052.281112:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879052.281115:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879052.281118:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879052.281120:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879052.281122:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879052.281124:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879052.281127:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9991 00000100:00080000:0.0:1788879052.281130:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879052.281132:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879052.281134:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879052.281137:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879052.281138:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879052.281139:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788879052.281140:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879052.281144:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879052.281145:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879052.281147:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879052.281149:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879052.281150:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879052.281154:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788879052.281156:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879052.281157:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879052.281160:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879052.281163:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879052.281163:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879052.281167:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879052.281168:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879052.281172:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879052.281173:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879052.281177:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:3.0:1788879052.281181:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290124672:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1788879052.281183:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879052.281189:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1788879052.281190:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879052.281194:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879052.281198:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879052.281199:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879052.281201:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879052.281201:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879052.281205:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879052.281206:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879052.281208:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879052.281209:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879052.281216:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879052.281216:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879052.281224:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290124800/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879052.281234:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879052.281236:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879052.281239:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879052.281240:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879052.281242:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879052.281243:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879052.281245:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879052.281246:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879052.281248:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9991 00000100:00080000:0.0:1788879052.281250:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879052.281252:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879052.281252:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879052.281254:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879052.281255:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879052.281256:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879052.281258:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879052.281259:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879052.281260:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879052.281262:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879052.281264:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879052.281265:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879052.281267:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879052.281268:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879052.281269:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879052.281271:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879052.281272:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879052.281273:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879052.281276:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879052.281277:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879052.281279:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879052.281280:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879052.281281:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879052.281284:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879052.281287:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290124928/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879052.281293:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879052.281294:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281295:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879052.281296:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281299:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00080000:0.0:1788879052.281301:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879052.281306:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788879052.281307:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879052.281310:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879052.281311:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879052.281314:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879052.281316:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9991 00000100:00080000:0.0:1788879052.281319:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879052.281321:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879052.281322:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879052.281324:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879052.281324:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879052.281326:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879052.281327:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879052.281329:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879052.281330:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879052.281333:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879052.281334:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879052.281336:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879052.281338:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:3.0:1788879052.281341:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:0.0:1788879052.281349:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281350:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879052.281351:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879052.281355:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879052.281356:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00000001:0.0:1788879052.281359:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281360:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879052.281363:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281364:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:0.0:1788879052.281369:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879052.281370:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879052.281373:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281375:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304664105 00000100:00000001:0.0:1788879052.281376:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879052.281380:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879052.281382:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879052.281386:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281387:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00100000:0.0:1788879052.281392:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290125056/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879052.281398:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879052.281401:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281401:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304664105 00000100:00000001:0.0:1788879052.281407:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281409:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879052.281414:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879052.281417:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281419:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879052.281424:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879052/real 1788879052] req@ffff94e508979c00 x1875765290124672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879052.281433:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879052.281435:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879052.281439:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879052.281441:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879052.281442:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.281444:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879052.281445:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879052.281449:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290124672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879052.281566:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.281568:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879052.281570:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.281572:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879052.281573:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.281575:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879052.281576:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879052.281579:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879052.281582:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879052.281584:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879052.281587:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290124672:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879052.281592:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.281594:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879052.281596:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879052.281631:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879052.281633:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879052.281636:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879052.281637:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879052.281640:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879052.281642:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879052.281645:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879052.281647:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.281649:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.281655:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879052.281657:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281659:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879052.281661:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879052.281663:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879052.281665:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879052.281666:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879052.281668:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879052.281671:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879052.281673:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879052.281676:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290124800:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879052.281678:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879052.281680:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879052.281681:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879052.281701:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879052.281706:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281716:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879052.281718:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:3.0:1788879052.281723:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879052.281745:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879052.281752:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879052.281755:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879052.281757:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281759:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879052.281763:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281768:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304663563 00000100:00000001:3.0:1788879052.281772:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304664105 00000100:00000001:3.0:1788879052.281775:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304663563 00000100:00000001:3.0:1788879052.281778:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304664105 00000100:00000001:3.0:1788879052.281781:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879052.281784:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281787:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879052.281792:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879052/real 1788879052] req@ffff94e50897ad80 x1875765290124800/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879052.281802:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879052.281803:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879052.281807:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879052.281809:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879052.281811:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.281813:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879052.281815:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879052.281820:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290124800/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879052.281828:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.281831:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879052.281835:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.281837:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879052.281839:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.281841:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879052.281871:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879052.281873:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879052.281875:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879052.281876:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879052.281878:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290124800:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879052.281881:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879052.281882:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879052.281883:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879052.281885:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879052.281886:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879052.281888:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879052.281889:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290124928:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879052.281891:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879052.281892:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879052.281893:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879052.281894:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879052.281896:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281900:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879052.281901:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:3.0:1788879052.281904:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879052.281911:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879052.281914:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879052.281915:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879052.281916:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281917:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879052.281920:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281923:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304663562 00000100:00000001:3.0:1788879052.281925:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304664105 00000100:00000001:3.0:1788879052.281927:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304663562 00000100:00000001:3.0:1788879052.281929:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304664105 00000100:00000001:3.0:1788879052.281930:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879052.281932:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.281953:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879052.281957:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879052/real 1788879052] req@ffff94e508978e00 x1875765290124928/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879052.281963:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879052.281964:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879052.281967:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879052.281968:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879052.281969:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.281971:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879052.281972:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879052.281975:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290124928/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879052.281979:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.281981:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879052.281982:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.281984:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879052.281985:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.281986:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879052.281996:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879052.281998:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879052.282001:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879052.282002:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879052.282005:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290124928:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879052.282010:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879052.282011:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879052.282012:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879052.282014:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879052.282015:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879052.282017:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879052.282019:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290125056:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879052.282021:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879052.282023:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879052.282024:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879052.282025:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879052.282028:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.282031:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879052.282032:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:3.0:1788879052.282034:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879052.282041:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879052.282044:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879052.282046:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879052.282047:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.282048:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879052.282050:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.282052:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304663562 00000100:00000001:3.0:1788879052.282054:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304664105 00000100:00000001:3.0:1788879052.282055:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304663562 00000100:00000001:3.0:1788879052.282057:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304664105 00000100:00000001:3.0:1788879052.282058:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879052.282060:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.282061:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879052.282063:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879052/real 1788879052] req@ffff94e50897bb80 x1875765290125056/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879052.282068:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879052.282068:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879052.282070:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879052.282071:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879052.282071:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.282072:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879052.282073:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879052.282075:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290125056/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879052.282080:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.282081:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879052.282083:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.282084:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879052.282085:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.282086:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879052.282094:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879052.282096:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879052.282098:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879052.282099:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879052.282101:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290125056:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879052.282104:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.282105:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879052.282106:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879052.282107:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879052.282108:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879052.282109:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879052.282110:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879052.282112:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879052.282113:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879052.282115:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879052.282116:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.282134:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879052.282136:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879052.282138:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879052.282140:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879052.282141:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879052.282143:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879052.282145:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879052.282147:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879052.282149:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879052.282151:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.282153:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879052.282154:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879052.282155:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879052.282157:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879052.282158:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879052.282160:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879052.282161:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879052.282162:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879052.282164:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879052.282166:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.282168:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879052.282172:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879052.282174:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879052.282176:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879052.282178:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879052.345006:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879052.345013:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879052.345016:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879052.345021:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879052.345026:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879052.345030:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879052.345033:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879052.345036:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879052.345039:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879053.049025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879053.049033:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879053.049039:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879053.049044:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879053.049047:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879053.049052:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879053.049055:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879053.049058:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879053.049061:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879053.049064:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879053.049159:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879053.049172:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879053.049181:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879053.369062:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879053.369071:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879053.369074:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879053.369082:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879053.369087:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879053.369093:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879053.369097:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879053.369101:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879053.369105:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879054.073017:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879054.073025:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879054.073032:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879054.073034:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879054.073056:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879054.073059:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879054.073062:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879054.393052:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879054.393065:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879054.393068:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879054.393077:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879054.393082:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879054.393088:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879054.393092:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879054.393097:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879054.393100:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879055.097052:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879055.097065:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879055.097069:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879055.097073:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879055.097081:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879055.097085:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879055.097088:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879055.097092:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879055.097097:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879055.097099:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879055.097128:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879055.097132:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879055.097136:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879055.417054:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879055.417063:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879055.417067:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879055.417074:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879055.417080:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879055.417100:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879055.417104:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879055.417108:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879055.417112:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879056.121055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879056.121067:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879056.121074:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879056.121076:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879056.121212:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879056.121220:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879056.121225:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879056.441131:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879056.441146:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879056.441150:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879056.441161:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879056.441169:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879056.441177:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879056.441183:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879056.441188:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879056.441193:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879057.145082:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879057.145090:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879057.145097:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879057.145099:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879057.145234:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879057.145243:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879057.145251:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879057.401110:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879057.401123:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879057.401127:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879057.401132:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879057.401138:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879057.401147:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 9996 00000100:00080000:2.0:1788879057.401160:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879057.401165:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879057.401168:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879057.401178:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879057.401182:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879057.401187:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879057.401192:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879057.401195:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879057.401197:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879057.401203:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879057.401208:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879057.401212:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879057.401218:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879057.401222:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879057.401225:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879057.401228:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879057.401243:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879057.401245:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879057.401249:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879057.401253:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879057.401258:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879057.401273:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879057.401285:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290125184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879057.401298:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879057.401304:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879057.401323:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879057.401328:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879057.401333:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879057.401336:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879057.401341:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879057.401348:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879057.401355:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9996 00000100:00080000:2.0:1788879057.401365:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879057.401370:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879057.401373:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879057.401384:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879057.401388:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879057.401393:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879057.401398:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879057.401402:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879057.401411:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879057.401412:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879057.401426:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788879057.401428:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879057.401432:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879057.401439:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.401440:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879057.401452:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879057.401454:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879057.401465:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879057.401469:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879057.401476:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788879057.401481:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:2.0:1788879057.401482:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788879057.401510:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879057.401518:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788879057.401520:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879057.401527:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879057.401536:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290125184:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879057.401539:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879057.401545:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.401552:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788879057.401553:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879057.401566:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879057.401570:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879057.401572:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879057.401577:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879057.401584:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879057.401585:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879057.401598:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879057.401599:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290125312/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879057.401623:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879057.401626:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879057.401633:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879057.401637:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879057.401639:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788879057.401643:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879057.401644:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00000001:2.0:1788879057.401653:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879057.401659:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879057.401661:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879057.401678:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879057.401683:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9996 00000100:00080000:2.0:1788879057.401691:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879057.401694:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879057.401697:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879057.401719:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879057.401723:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879057.401726:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879057.401729:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879057.401733:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879057.401736:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879057.401743:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879057.401746:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879057.401749:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879057.401757:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:3.0:1788879057.401758:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:2.0:1788879057.401770:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879057.401776:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879057.401779:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879057.401785:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879057.401789:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.401793:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:2.0:1788879057.401796:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879057.401800:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.401806:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879057.401808:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879057.401822:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879057.401824:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.401831:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:2.0:1788879057.401832:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879057.401849:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.401850:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.401871:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304669225 00000100:00100000:2.0:1788879057.401873:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290125440/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879057.401890:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879057.401893:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.401896:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788879057.401899:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879057.401945:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879057.401949:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879057.401951:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879057.401953:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879057.401957:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879057.401962:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 9996 00000100:00080000:2.0:1788879057.401967:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879057.401970:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879057.401973:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304669226 00000100:00000001:2.0:1788879057.401974:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879057.401984:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879057.401987:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879057.401999:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879057.402003:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1788879057.402005:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879057.402020:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879057.402024:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879057.402025:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879057.402047:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879057.402050:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879057.402062:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879057.402063:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879057.402074:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879057.402080:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879057.402084:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879057.402091:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:3.0:1788879057.402092:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879057/real 1788879057] req@ffff94e5045e7480 x1875765290125184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788879057.402094:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879057.402116:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879057.402128:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879057.402147:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879057.402152:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879057.402166:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879057.402175:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879057.402188:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879057.402189:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879057.402201:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879057.402206:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.402211:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879057.402216:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00100000:2.0:1788879057.402219:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290125568/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879057.402221:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402238:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879057.402248:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879057.402254:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879057.402260:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879057.402264:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290125184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879057.402268:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.402292:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402299:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879057.402304:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402307:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879057.402311:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402316:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879057.402319:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879057.402324:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879057.402329:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879057.402332:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879057.402337:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290125184:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879057.402345:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402348:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879057.402350:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879057.402354:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879057.402359:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879057.402363:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879057.402366:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879057.402372:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879057.402374:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879057.402379:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879057.402381:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402385:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402396:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879057.402399:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.402402:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879057.402405:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879057.402407:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879057.402410:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879057.402412:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879057.402415:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879057.402418:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879057.402422:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879057.402426:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290125312:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879057.402430:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879057.402433:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879057.402435:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879057.402437:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879057.402441:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.402449:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879057.402451:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:3.0:1788879057.402458:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879057.402477:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879057.402482:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879057.402485:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879057.402488:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.402490:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879057.402516:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.402523:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304668684 00000100:00000001:3.0:1788879057.402527:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304669225 00000100:00000001:3.0:1788879057.402531:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304668684 00000100:00000001:3.0:1788879057.402534:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304669226 00000100:00000001:3.0:1788879057.402537:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879057.402539:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.402541:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879057.402547:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879057/real 1788879057] req@ffff94e5045e4700 x1875765290125312/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879057.402567:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879057.402569:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879057.402573:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879057.402575:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879057.402577:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402579:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879057.402581:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879057.402586:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290125312/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879057.402595:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402597:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879057.402600:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402602:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879057.402603:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402606:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879057.402628:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879057.402631:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879057.402636:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879057.402639:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879057.402643:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290125312:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879057.402649:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879057.402651:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879057.402652:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879057.402656:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879057.402658:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879057.402660:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879057.402664:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290125440:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879057.402667:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879057.402670:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879057.402673:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879057.402675:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879057.402679:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.402686:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879057.402688:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:3.0:1788879057.402693:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879057.402707:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879057.402712:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879057.402715:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879057.402716:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.402719:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879057.402722:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.402726:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304668682 00000100:00000001:3.0:1788879057.402729:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304669225 00000100:00000001:3.0:1788879057.402732:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304668682 00000100:00000001:3.0:1788879057.402736:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304669226 00000100:00000001:3.0:1788879057.402738:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879057.402740:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.402742:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879057.402747:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879057/real 1788879057] req@ffff94e5045e6680 x1875765290125440/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879057.402757:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879057.402759:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879057.402762:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879057.402764:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879057.402766:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402768:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879057.402769:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879057.402774:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290125440/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879057.402782:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402784:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879057.402788:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402790:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879057.402792:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402794:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879057.402813:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879057.402816:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879057.402821:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879057.402823:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879057.402827:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290125440:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879057.402832:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879057.402833:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879057.402835:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879057.402838:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879057.402840:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879057.402842:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879057.402845:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290125568:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879057.402850:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879057.402852:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879057.402854:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879057.402856:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879057.402860:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.402865:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879057.402867:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:3.0:1788879057.402871:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879057.402882:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879057.402886:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879057.402888:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879057.402890:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.402892:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879057.402895:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.402900:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304668682 00000100:00000001:3.0:1788879057.402933:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304669225 00000100:00000001:3.0:1788879057.402938:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304668683 00000100:00000001:3.0:1788879057.402941:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304669226 00000100:00000001:3.0:1788879057.402944:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879057.402947:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.402950:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879057.402955:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879057/real 1788879057] req@ffff94e5045e6a00 x1875765290125568/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879057.402964:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879057.402965:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879057.402969:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879057.402971:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879057.402973:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402976:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879057.402978:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879057.402982:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290125568/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879057.402991:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402994:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879057.402996:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.402998:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879057.403000:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.403002:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879057.403021:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879057.403024:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879057.403027:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879057.403030:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879057.403033:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290125568:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879057.403037:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.403040:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879057.403042:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879057.403044:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879057.403047:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879057.403049:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879057.403050:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879057.403054:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879057.403055:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879057.403058:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879057.403060:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.403062:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879057.403064:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879057.403066:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879057.403068:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879057.403070:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879057.403071:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879057.403073:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879057.403075:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879057.403077:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879057.403079:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.403081:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879057.403082:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879057.403084:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879057.403085:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879057.403086:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879057.403088:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879057.403089:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879057.403091:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879057.403092:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879057.403094:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.403096:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879057.403099:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879057.403101:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879057.403103:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879057.403105:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879057.464994:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879057.465002:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879057.465005:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879057.465012:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879057.465016:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879057.465021:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879057.465024:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879057.465027:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879057.465031:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879058.169100:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879058.169109:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879058.169115:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879058.169117:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879058.169140:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879058.169144:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879058.169147:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879058.489048:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879058.489061:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879058.489067:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879058.489079:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879058.489087:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879058.489094:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879058.489099:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879058.489105:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879058.489127:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879059.193130:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879059.193141:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879059.193149:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879059.193151:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879059.193179:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879059.193183:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879059.193187:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879059.513036:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879059.513044:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879059.513047:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879059.513055:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879059.513061:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879059.513067:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879059.513071:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879059.513075:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879059.513079:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879060.217067:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879060.217077:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879060.217084:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879060.217086:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879060.217226:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879060.217234:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879060.217240:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879060.537084:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879060.537094:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879060.537097:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879060.537104:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879060.537109:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879060.537114:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879060.537118:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879060.537121:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879060.537124:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879061.241090:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879061.241100:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879061.241107:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879061.241109:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879061.241217:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879061.241222:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879061.241228:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879061.561017:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879061.561026:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879061.561029:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879061.561036:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879061.561041:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879061.561046:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879061.561049:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879061.561053:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879061.561056:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879062.265145:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879062.265156:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879062.265163:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879062.265165:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879062.265210:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879062.265214:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879062.265218:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879062.521167:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879062.521181:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879062.521186:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879062.521192:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879062.521199:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879062.521211:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10001 00000100:00080000:0.0:1788879062.521229:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879062.521233:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879062.521237:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879062.521248:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879062.521253:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879062.521258:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879062.521261:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879062.521264:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879062.521266:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879062.521273:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879062.521276:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879062.521280:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879062.521285:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879062.521290:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879062.521293:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879062.521296:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879062.521317:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879062.521319:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879062.521323:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879062.521329:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879062.521337:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879062.521357:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879062.521371:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290125696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879062.521386:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879062.521393:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879062.521423:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879062.521428:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879062.521432:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879062.521435:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879062.521437:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879062.521441:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879062.521444:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10001 00000100:00080000:0.0:1788879062.521449:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879062.521452:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879062.521454:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879062.521489:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879062.521492:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879062.521496:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879062.521501:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879062.521504:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879062.521515:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879062.521520:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879062.521523:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879062.521527:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879062.521530:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788879062.521537:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879062.521542:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788879062.521545:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.521549:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879062.521551:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879062.521561:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:0.0:1788879062.521564:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879062.521567:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788879062.521575:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879062.521584:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879062.521585:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879062.521598:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879062.521601:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879062.521605:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879062.521615:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879062.521616:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879062.521618:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879062.521623:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879062.521636:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879062.521647:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290125696:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1788879062.521649:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979180 x1875765290125824/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879062.521682:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788879062.521685:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879062.521693:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879062.521698:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788879062.521700:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879062.521710:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1788879062.521711:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879062.521730:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:3.0:1788879062.521732:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879062.521735:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879062.521739:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879062.521744:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879062.521749:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10001 02000000:00000001:3.0:1788879062.521750:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879062.521755:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879062.521761:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879062.521763:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879062.521767:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879062.521769:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879062.521773:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879062.521776:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879062.521779:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879062.521781:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879062.521786:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879062.521788:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879062.521792:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.521793:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 02000000:00000001:0.0:1788879062.521805:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879062.521809:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1788879062.521812:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879062.521818:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879062.521822:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879062.521824:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879062.521828:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879062.521830:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879062.521835:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879062.521837:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879062.521840:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879062.521843:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879062.521846:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879062.521852:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879062.521863:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290125952/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879062.521873:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879062.521875:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879062.521881:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879062.521891:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788879062.521896:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1788879062.521898:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879062.521967:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879062.521969:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879062.521974:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879062.521979:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10001 00000100:00080000:0.0:1788879062.521984:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879062.521987:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879062.521992:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879062.522000:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879062.522004:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879062.522007:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879062.522010:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879062.522014:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879062.522016:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879062.522018:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:0.0:1788879062.522027:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879062.522032:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879062.522038:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879062.522039:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1788879062.522044:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879062.522048:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879062.522052:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.522065:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879062.522066:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879062.522075:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879062.522081:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879062.522084:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.522088:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879062.522093:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879062.522102:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879062.522106:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879062.522109:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304674346 02000000:00000001:0.0:1788879062.522111:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879062.522117:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879062.522133:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879062.522141:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290126080/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879062.522143:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788879062.522164:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879062.522168:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.522171:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304674346 00000100:00000001:0.0:1788879062.522175:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.522205:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879062.522218:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879062.522226:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.522232:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879062.522250:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879062/real 1788879062] req@ffff94e508978380 x1875765290125696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879062.522280:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879062.522283:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879062.522290:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879062.522294:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879062.522297:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.522300:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879062.522303:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879062.522310:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290125696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879062.522321:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.522326:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879062.522330:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.522332:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879062.522333:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.522337:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879062.522340:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879062.522344:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879062.522350:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879062.522354:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879062.522359:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290125696:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879062.522365:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.522368:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879062.522370:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879062.522373:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879062.522378:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879062.522381:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879062.522384:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879062.522390:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879062.522392:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879062.522398:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879062.522401:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.522405:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.522417:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879062.522420:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.522425:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879062.522428:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879062.522431:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879062.522434:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879062.522437:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879062.522443:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879062.522447:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879062.522452:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879062.522486:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290125824:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879062.522494:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879062.522500:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879062.522503:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879062.522508:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879062.522524:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.522538:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879062.522541:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 00000100:00000001:3.0:1788879062.522550:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879062.522582:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879062.522592:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879062.522598:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879062.522600:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.522603:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879062.522607:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.522614:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304673804 00000100:00000001:3.0:1788879062.522618:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304674346 00000100:00000001:3.0:1788879062.522621:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304673804 00000100:00000001:3.0:1788879062.522624:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304674346 00000100:00000001:3.0:1788879062.522627:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879062.522630:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.522632:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879062.522639:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879062/real 1788879062] req@ffff94e508979180 x1875765290125824/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879062.522650:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879062.522652:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879062.522656:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879062.522658:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879062.522660:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.522662:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879062.522664:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879062.522671:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290125824/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879062.522697:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.522702:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879062.522707:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.522710:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879062.522713:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.522717:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879062.522739:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879062.522746:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879062.522754:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879062.522759:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879062.522767:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290125824:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879062.522775:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879062.522779:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879062.522782:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879062.522787:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879062.522790:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879062.522795:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879062.522801:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290125952:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879062.522806:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879062.522811:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879062.522814:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879062.522818:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879062.522823:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.522832:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879062.522834:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000100:00000001:3.0:1788879062.522840:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879062.522860:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879062.522866:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879062.522870:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879062.522872:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.522875:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879062.522879:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.522885:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304673802 00000100:00000001:3.0:1788879062.522890:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304674346 00000100:00000001:3.0:1788879062.522894:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304673802 00000100:00000001:3.0:1788879062.522899:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304674346 00000100:00000001:3.0:1788879062.522943:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879062.522948:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.522951:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879062.522958:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879062/real 1788879062] req@ffff94e50897b800 x1875765290125952/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879062.522969:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879062.522971:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879062.522976:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879062.522978:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879062.522980:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.522982:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879062.522984:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879062.522989:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290125952/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879062.522998:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.523001:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879062.523004:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.523006:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879062.523007:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.523010:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879062.523024:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879062.523028:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879062.523031:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879062.523034:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879062.523039:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290125952:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879062.523044:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879062.523046:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879062.523048:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879062.523052:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879062.523054:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879062.523056:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879062.523060:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290126080:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879062.523064:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879062.523067:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879062.523069:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879062.523072:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879062.523076:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.523082:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879062.523084:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:3.0:1788879062.523089:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879062.523107:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879062.523116:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879062.523120:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879062.523123:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.523127:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879062.523132:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.523139:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304673803 00000100:00000001:3.0:1788879062.523144:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304674346 00000100:00000001:3.0:1788879062.523148:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304673803 00000100:00000001:3.0:1788879062.523152:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304674346 00000100:00000001:3.0:1788879062.523156:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879062.523160:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.523163:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879062.523172:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879062/real 1788879062] req@ffff94e50897bb80 x1875765290126080/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879062.523203:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879062.523206:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879062.523211:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879062.523215:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879062.523218:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.523222:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879062.523225:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879062.523234:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290126080/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879062.523243:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.523246:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879062.523249:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.523251:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879062.523253:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.523255:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879062.523269:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879062.523272:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879062.523276:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879062.523279:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879062.523283:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290126080:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879062.523288:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.523291:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879062.523293:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879062.523295:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879062.523299:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879062.523301:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879062.523302:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879062.523306:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879062.523308:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879062.523312:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879062.523313:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.523316:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879062.523318:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879062.523319:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879062.523321:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879062.523323:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879062.523325:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879062.523327:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879062.523328:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879062.523331:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879062.523332:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.523334:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879062.523336:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879062.523337:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879062.523339:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879062.523340:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879062.523341:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879062.523343:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879062.523345:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879062.523346:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879062.523348:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.523350:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879062.523355:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879062.523357:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879062.523359:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879062.523361:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879062.585043:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879062.585053:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879062.585057:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879062.585067:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879062.585073:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879062.585079:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879062.585084:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879062.585089:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879062.585094:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879063.289030:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879063.289037:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879063.289042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879063.289045:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879063.289048:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879063.289052:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879063.289071:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879063.289074:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879063.289077:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879063.289081:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879063.289112:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879063.289115:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879063.289119:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879063.608982:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879063.608988:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879063.608990:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879063.608996:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879063.609000:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879063.609004:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879063.609006:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879063.609009:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879063.609012:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879064.313105:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879064.313115:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879064.313124:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879064.313126:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879064.313239:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879064.313248:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879064.313254:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879064.633015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879064.633023:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879064.633026:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879064.633032:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879064.633038:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879064.633043:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879064.633046:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879064.633049:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879064.633053:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879065.337035:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879065.337044:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879065.337048:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879065.337052:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879065.337059:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879065.337062:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879065.337064:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879065.337067:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879065.337072:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879065.337073:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879065.337095:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879065.337098:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879065.337101:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879065.657063:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879065.657072:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879065.657074:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879065.657081:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879065.657085:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879065.657090:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879065.657094:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879065.657097:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879065.657100:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879066.361098:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879066.361109:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879066.361118:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879066.361121:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879066.361154:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879066.361159:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879066.361164:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879066.680996:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879066.681005:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879066.681008:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879066.681016:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879066.681022:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879066.681027:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879066.681032:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879066.681036:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879066.681041:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879067.385075:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879067.385088:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879067.385099:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879067.385102:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879067.385141:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879067.385148:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879067.385155:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879067.641033:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879067.641063:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879067.641068:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879067.641075:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879067.641082:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879067.641093:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10006 00000100:00080000:0.0:1788879067.641110:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879067.641115:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879067.641119:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879067.641132:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879067.641136:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879067.641141:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879067.641144:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879067.641147:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879067.641149:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879067.641155:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879067.641158:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879067.641163:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879067.641167:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879067.641172:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879067.641175:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879067.641178:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879067.641195:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879067.641197:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879067.641201:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879067.641206:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879067.641213:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879067.641232:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879067.641246:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290126208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879067.641262:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879067.641269:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879067.641288:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879067.641293:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879067.641298:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879067.641300:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879067.641303:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879067.641306:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879067.641310:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10006 00000100:00080000:0.0:1788879067.641315:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879067.641319:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879067.641322:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879067.641326:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879067.641329:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879067.641332:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879067.641336:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879067.641339:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879067.641341:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879067.641346:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879067.641349:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879067.641350:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879067.641359:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879067.641364:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788879067.641369:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879067.641374:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.641374:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879067.641383:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879067.641385:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:0.0:1788879067.641390:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879067.641394:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788879067.641396:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879067.641406:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879067.641412:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879067.641414:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879067.641445:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879067.641450:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879067.641452:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879067.641455:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879067.641465:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879067.641472:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290126336/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879067.641492:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879067.641495:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879067.641499:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879067.641503:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879067.641508:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879067.641510:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879067.641512:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879067.641517:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879067.641520:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10006 00000100:00080000:0.0:1788879067.641525:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879067.641528:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879067.641530:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879067.641534:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879067.641536:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879067.641539:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879067.641542:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879067.641545:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879067.641547:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:3.0:1788879067.641552:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1788879067.641553:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:3.0:1788879067.641566:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879067.641567:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879067.641578:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879067.641582:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788879067.641586:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290126208:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788879067.641587:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879067.641595:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879067.641601:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879067.641604:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1788879067.641606:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879067.641612:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879067.641613:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879067.641619:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879067.641621:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879067.641624:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879067.641630:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879067.641631:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879067.641636:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879067.641639:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879067.641642:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879067.641648:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879067.641659:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290126464/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879067.641670:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879067.641672:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.641675:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879067.641677:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.641681:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00080000:0.0:1788879067.641684:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879067.641695:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788879067.641696:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879067.641703:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879067.641707:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879067.641714:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879067.641719:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10006 00000100:00080000:0.0:1788879067.641728:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879067.641732:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879067.641734:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879067.641757:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879067.641761:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:3.0:1788879067.641768:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:0.0:1788879067.641770:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879067.641779:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879067.641784:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879067.641789:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879067.641789:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879067.641798:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879067.641804:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879067.641806:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879067.641809:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879067.641817:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.641821:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879067.641822:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879067.641831:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879067.641833:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304679465 02000000:00000001:0.0:1788879067.641837:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879067.641843:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879067.641846:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879067.641850:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879067.641852:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.641855:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1788879067.641861:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879067.641866:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.641869:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304679465 00000100:00000001:0.0:1788879067.641870:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879067.641875:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879067.641880:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.641884:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788879067.641891:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.641893:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879067.641933:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.641938:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879067.641949:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879067/real 1788879067] req@ffff94e508978e00 x1875765290126208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879067.641953:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290126592/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788879067.642017:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788879067.642019:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:3.0:1788879067.642027:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879067.642028:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879067.642037:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.642044:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879067.642054:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879067.642057:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642065:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879067.642070:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879067.642083:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290126208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879067.642100:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642107:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879067.642111:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642113:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879067.642116:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642120:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879067.642123:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879067.642127:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879067.642133:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879067.642137:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879067.642142:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290126208:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879067.642149:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642152:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879067.642155:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879067.642158:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879067.642163:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879067.642166:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879067.642169:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879067.642175:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879067.642177:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879067.642183:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879067.642185:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642188:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642198:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879067.642202:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.642204:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879067.642207:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879067.642209:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879067.642211:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879067.642213:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879067.642216:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879067.642219:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879067.642222:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879067.642226:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290126336:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879067.642230:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879067.642233:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879067.642235:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879067.642237:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879067.642242:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.642253:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879067.642255:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:3.0:1788879067.642262:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879067.642282:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879067.642288:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879067.642291:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879067.642293:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.642295:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879067.642299:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.642305:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304678924 00000100:00000001:3.0:1788879067.642309:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304679465 00000100:00000001:3.0:1788879067.642312:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304678924 00000100:00000001:3.0:1788879067.642315:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304679465 00000100:00000001:3.0:1788879067.642318:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879067.642320:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.642323:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879067.642328:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879067/real 1788879067] req@ffff94e50897ad80 x1875765290126336/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879067.642344:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879067.642347:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879067.642351:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879067.642354:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879067.642357:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642360:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879067.642363:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879067.642371:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290126336/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879067.642387:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642392:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879067.642397:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642400:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879067.642403:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642408:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879067.642460:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879067.642465:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879067.642470:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879067.642473:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879067.642478:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290126336:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879067.642483:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879067.642486:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879067.642488:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879067.642491:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879067.642494:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879067.642496:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879067.642500:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290126464:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879067.642505:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879067.642508:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879067.642510:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879067.642513:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879067.642518:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.642528:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879067.642530:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:3.0:1788879067.642536:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879067.642554:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879067.642561:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879067.642564:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879067.642566:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.642568:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879067.642571:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.642576:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304678922 00000100:00000001:3.0:1788879067.642579:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304679465 00000100:00000001:3.0:1788879067.642582:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304678922 00000100:00000001:3.0:1788879067.642585:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304679465 00000100:00000001:3.0:1788879067.642591:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879067.642593:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.642595:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879067.642601:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879067/real 1788879067] req@ffff94e508979c00 x1875765290126464/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879067.642612:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879067.642614:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879067.642617:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879067.642619:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879067.642621:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642623:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879067.642624:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879067.642629:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290126464/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879067.642638:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642641:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879067.642644:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642646:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879067.642648:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642650:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879067.642668:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879067.642671:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879067.642674:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879067.642677:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879067.642681:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290126464:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879067.642685:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879067.642687:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879067.642689:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879067.642692:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879067.642694:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879067.642697:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879067.642701:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290126592:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879067.642705:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879067.642707:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879067.642709:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879067.642712:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879067.642715:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.642720:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879067.642722:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:3.0:1788879067.642726:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879067.642737:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879067.642741:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879067.642744:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879067.642746:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.642748:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879067.642750:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.642754:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304678922 00000100:00000001:3.0:1788879067.642757:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304679465 00000100:00000001:3.0:1788879067.642760:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304678922 00000100:00000001:3.0:1788879067.642763:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304679465 00000100:00000001:3.0:1788879067.642766:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879067.642768:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.642770:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879067.642775:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879067/real 1788879067] req@ffff94e50897a300 x1875765290126592/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879067.642783:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879067.642785:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879067.642787:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879067.642789:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879067.642790:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642792:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879067.642794:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879067.642798:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290126592/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879067.642816:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642818:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879067.642821:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642823:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879067.642825:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642827:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879067.642845:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879067.642847:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879067.642851:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879067.642853:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879067.642857:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290126592:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879067.642862:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642864:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879067.642866:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879067.642868:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879067.642871:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879067.642873:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879067.642874:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879067.642878:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879067.642880:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879067.642885:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879067.642887:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642889:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879067.642891:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879067.642892:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879067.642895:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879067.642896:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879067.642898:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879067.642900:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879067.642902:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879067.642904:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879067.642935:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642938:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879067.642939:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879067.642942:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879067.642944:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879067.642946:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879067.642947:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879067.642950:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879067.642951:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879067.642954:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879067.642955:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642958:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879067.642962:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879067.642964:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879067.642966:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879067.642969:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879067.705080:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879067.705093:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879067.705096:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879067.705104:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879067.705111:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879067.705117:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879067.705121:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879067.705125:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879067.705129:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879068.409029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879068.409037:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879068.409044:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879068.409046:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879068.409140:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879068.409146:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879068.409150:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879068.729043:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879068.729054:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879068.729058:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879068.729067:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879068.729073:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879068.729079:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879068.729084:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879068.729089:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879068.729094:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879069.433033:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879069.433041:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879069.433066:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879069.433068:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879069.433096:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879069.433100:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879069.433104:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879069.753011:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879069.753020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879069.753024:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879069.753032:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879069.753037:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879069.753042:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879069.753046:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879069.753050:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879069.753054:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879070.457020:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879070.457027:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879070.457033:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879070.457047:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879070.457090:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879070.457094:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879070.457097:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879070.777004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879070.777017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879070.777020:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879070.777028:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879070.777035:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879070.777042:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879070.777048:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879070.777053:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879070.777059:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879071.481121:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879071.481127:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879071.481132:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879071.481133:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879071.481159:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879071.481162:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879071.481164:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879071.801000:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879071.801012:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879071.801015:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879071.801025:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879071.801032:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879071.801040:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879071.801047:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879071.801053:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879071.801059:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879072.505105:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879072.505114:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879072.505124:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879072.505126:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879072.505233:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879072.505242:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879072.505248:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879072.761095:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879072.761113:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879072.761117:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879072.761123:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879072.761130:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879072.761153:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10011 00000100:00080000:2.0:1788879072.761189:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879072.761194:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879072.761197:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879072.761213:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879072.761220:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879072.761228:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879072.761244:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879072.761249:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879072.761252:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879072.761263:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879072.761270:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879072.761278:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879072.761285:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879072.761292:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879072.761297:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879072.761337:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879072.761352:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879072.761354:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879072.761358:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879072.761361:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879072.761367:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879072.761382:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879072.761410:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290126720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879072.761422:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879072.761427:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879072.761443:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879072.761447:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879072.761450:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879072.761452:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879072.761453:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879072.761456:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879072.761460:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10011 00000100:00080000:2.0:1788879072.761466:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879072.761468:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879072.761470:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879072.761471:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879072.761477:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879072.761478:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788879072.761479:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879072.761482:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788879072.761484:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879072.761487:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1788879072.761488:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879072.761490:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879072.761492:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879072.761496:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:3.0:1788879072.761497:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1788879072.761501:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:3.0:1788879072.761503:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879072.761505:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879072.761509:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879072.761510:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1788879072.761518:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290126720:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879072.761519:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879072.761523:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879072.761530:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879072.761535:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:2.0:1788879072.761536:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788879072.761540:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:2.0:1788879072.761541:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879072.761545:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879072.761546:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879072.761552:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879072.761555:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879072.761558:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879072.761559:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879072.761560:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879072.761565:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879072.761568:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879072.761576:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879072.761581:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290126848/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879072.761597:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879072.761601:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:2.0:1788879072.761602:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879072.761608:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.761609:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879072.761612:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879072.761615:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879072.761619:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879072.761621:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879072.761623:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879072.761625:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879072.761646:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10011 00000100:00080000:2.0:1788879072.761671:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879072.761676:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879072.761679:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879072.761687:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879072.761691:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:3.0:1788879072.761703:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:2.0:1788879072.761706:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879072.761715:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879072.761722:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879072.761725:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879072.761732:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788879072.761736:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879072.761745:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879072.761747:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.761754:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879072.761756:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879072.761765:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:2.0:1788879072.761766:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879072.761782:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879072.761785:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879072.761790:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788879072.761791:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879072.761802:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879072.761805:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.761807:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304684585 02000000:00000001:2.0:1788879072.761812:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879072.761817:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879072.761820:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879072.761822:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879072.761824:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.761831:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788879072.761835:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879072.761846:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290126976/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879072.761855:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304684585 00000100:00000001:2.0:1788879072.761858:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879072.761864:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.761866:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879072.761870:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.761874:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788879072.761876:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879072.761883:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879072.761885:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879072.761891:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879072.761892:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879072.761900:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879072.761953:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879072.761958:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10011 00000100:00100000:3.0:1788879072.761974:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879072/real 1788879072] req@ffff94e5045e5880 x1875765290126720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879072.761976:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879072.761984:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879072.761990:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879072.762000:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879072.762006:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879072.762010:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000400:00000001:3.0:1788879072.762011:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1788879072.762014:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000400:00000001:3.0:1788879072.762016:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879072.762024:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879072.762027:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879072.762029:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:2.0:1788879072.762036:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879072.762038:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879072.762043:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762044:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879072.762053:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879072.762078:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1788879072.762084:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879072.762086:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879072.762088:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879072.762092:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:3.0:1788879072.762095:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290126720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1788879072.762096:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879072.762109:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762113:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:2.0:1788879072.762114:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762117:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879072.762118:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762121:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879072.762123:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762126:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879072.762127:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788879072.762129:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00080000:3.0:1788879072.762130:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1788879072.762133:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879072.762137:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879072.762138:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879072.762159:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1788879072.762160:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879072.762167:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879072.762172:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290126720:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879072.762179:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762182:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762188:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879072.762191:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00100000:2.0:1788879072.762196:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290127104/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879072.762197:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879072.762204:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879072.762208:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879072.762214:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879072.762220:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1788879072.762225:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788879072.762225:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879072.762233:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879072.762236:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762237:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762247:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879072.762252:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762259:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879072.762265:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762269:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879072.762272:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879072.762275:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879072.762277:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879072.762278:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879072.762282:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879072.762295:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879072.762299:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879072.762304:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290126848:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879072.762309:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879072.762312:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879072.762314:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879072.762316:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879072.762320:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762330:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879072.762332:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 00000100:00000001:3.0:1788879072.762339:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879072.762369:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879072.762374:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879072.762376:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879072.762377:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762379:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879072.762381:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762406:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304684044 00000100:00000001:3.0:1788879072.762410:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304684585 00000100:00000001:3.0:1788879072.762413:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304684044 00000100:00000001:3.0:1788879072.762414:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304684585 00000100:00000001:3.0:1788879072.762416:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879072.762418:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762419:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879072.762423:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879072/real 1788879072] req@ffff94e5045e5180 x1875765290126848/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879072.762430:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879072.762431:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879072.762434:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879072.762435:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879072.762436:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762438:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879072.762439:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879072.762443:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290126848/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879072.762448:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762450:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879072.762451:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762453:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879072.762454:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762455:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879072.762472:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879072.762474:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879072.762477:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879072.762479:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879072.762483:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290126848:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879072.762486:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879072.762487:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879072.762489:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879072.762490:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879072.762492:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879072.762493:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879072.762496:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290126976:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879072.762499:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879072.762500:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879072.762502:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879072.762503:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879072.762506:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762511:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879072.762512:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:3.0:1788879072.762528:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879072.762538:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879072.762542:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879072.762544:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879072.762545:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762546:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879072.762548:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762551:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304684042 00000100:00000001:3.0:1788879072.762553:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304684585 00000100:00000001:3.0:1788879072.762556:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304684042 00000100:00000001:3.0:1788879072.762558:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304684585 00000100:00000001:3.0:1788879072.762560:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879072.762562:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762563:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879072.762566:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879072/real 1788879072] req@ffff94e5045e4a80 x1875765290126976/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879072.762577:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879072.762578:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879072.762580:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879072.762582:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879072.762583:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762584:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879072.762593:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879072.762595:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290126976/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879072.762600:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762602:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879072.762603:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762605:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879072.762606:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762607:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879072.762622:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879072.762624:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879072.762626:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879072.762627:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879072.762629:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290126976:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879072.762632:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879072.762650:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879072.762651:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879072.762654:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879072.762656:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879072.762659:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879072.762664:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290127104:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879072.762672:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879072.762676:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879072.762679:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879072.762683:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879072.762688:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762699:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879072.762702:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:3.0:1788879072.762709:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879072.762733:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879072.762741:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879072.762746:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879072.762749:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762752:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879072.762758:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762764:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304684042 00000100:00000001:3.0:1788879072.762769:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304684585 00000100:00000001:3.0:1788879072.762774:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304684042 00000100:00000001:3.0:1788879072.762778:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304684585 00000100:00000001:3.0:1788879072.762782:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879072.762787:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.762790:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879072.762798:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879072/real 1788879072] req@ffff94e5045e7100 x1875765290127104/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879072.762813:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879072.762815:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879072.762820:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879072.762823:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879072.762826:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762829:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879072.762832:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879072.762839:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290127104/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879072.762852:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762856:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879072.762861:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762864:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879072.762867:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762871:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879072.762894:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879072.762900:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879072.762906:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879072.762910:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879072.762950:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290127104:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879072.762959:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762963:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879072.762964:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879072.762967:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879072.762970:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879072.762972:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879072.762974:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879072.762990:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879072.762992:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879072.762994:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879072.762996:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.762998:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879072.763000:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879072.763001:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879072.763004:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879072.763005:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879072.763006:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879072.763008:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879072.763009:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879072.763011:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879072.763013:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.763015:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879072.763016:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879072.763017:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879072.763019:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879072.763020:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879072.763021:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879072.763023:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879072.763026:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879072.763028:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879072.763030:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.763032:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879072.763037:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879072.763039:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879072.763042:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879072.763044:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879072.825004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879072.825015:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879072.825018:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879072.825025:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879072.825030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879072.825035:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879072.825039:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879072.825043:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879072.825047:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879073.529088:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879073.529099:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879073.529107:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879073.529112:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879073.529115:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879073.529120:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879073.529123:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879073.529126:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879073.529129:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879073.529132:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879073.529160:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879073.529164:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879073.529167:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879073.849010:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879073.849021:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879073.849024:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879073.849032:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879073.849038:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879073.849043:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879073.849047:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879073.849051:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879073.849054:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879074.553034:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879074.553043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879074.553051:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879074.553053:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879074.553078:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879074.553082:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879074.553086:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879074.872994:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879074.873003:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879074.873006:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879074.873015:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879074.873022:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879074.873028:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879074.873033:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879074.873037:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879074.873042:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879075.577063:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879075.577075:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879075.577080:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879075.577084:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879075.577092:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879075.577095:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879075.577098:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879075.577101:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879075.577106:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879075.577108:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879075.577133:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879075.577137:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879075.577140:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879075.896978:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879075.896985:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879075.896986:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879075.896991:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879075.896994:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879075.896998:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879075.897000:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879075.897002:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879075.897006:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879076.601080:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879076.601103:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879076.601112:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879076.601113:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879076.601256:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879076.601262:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879076.601266:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879076.921049:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879076.921072:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879076.921075:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879076.921084:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879076.921089:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879076.921095:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879076.921099:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879076.921103:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879076.921107:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879077.625040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879077.625051:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879077.625058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879077.625060:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879077.625088:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879077.625092:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879077.625096:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879077.881071:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879077.881086:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879077.881090:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879077.881096:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879077.881102:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879077.881113:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10016 00000100:00080000:0.0:1788879077.881129:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879077.881133:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879077.881136:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879077.881148:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879077.881151:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879077.881157:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879077.881160:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879077.881163:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879077.881165:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879077.881171:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879077.881174:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879077.881178:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879077.881183:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879077.881187:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879077.881190:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879077.881193:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879077.881210:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879077.881212:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879077.881216:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879077.881220:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879077.881227:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879077.881244:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879077.881258:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290127232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879077.881293:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879077.881301:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879077.881335:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879077.881343:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879077.881379:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879077.881383:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879077.881387:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879077.881393:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879077.881398:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10016 00000100:00080000:0.0:1788879077.881406:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879077.881409:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879077.881412:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879077.881417:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879077.881420:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879077.881424:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879077.881427:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879077.881430:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879077.881432:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879077.881434:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1788879077.881438:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879077.881445:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.881447:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788879077.881448:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879077.881459:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1788879077.881461:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879077.881465:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788879077.881468:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.881473:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879077.881474:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879077.881483:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788879077.881487:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:0.0:1788879077.881491:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879077.881497:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879077.881499:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879077.881511:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879077.881518:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1788879077.881525:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290127232:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788879077.881527:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879077.881540:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879077.881541:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879077.881550:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879077.881551:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879077.881558:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879077.881560:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879077.881564:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879077.881578:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879077.881579:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879077.881614:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978700 x1875765290127360/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879077.881617:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879077.881625:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:0.0:1788879077.881638:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879077.881641:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879077.881645:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879077.881654:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879077.881658:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879077.881662:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879077.881665:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879077.881667:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879077.881672:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879077.881677:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10016 00000100:00080000:0.0:1788879077.881684:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879077.881686:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879077.881689:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879077.881696:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879077.881698:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879077.881701:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879077.881704:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879077.881708:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879077.881712:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:3.0:1788879077.881716:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1788879077.881723:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879077.881729:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879077.881733:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879077.881737:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879077.881739:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:0.0:1788879077.881741:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.881749:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879077.881750:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879077.881755:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788879077.881759:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879077.881760:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000008:00000001:0.0:1788879077.881770:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.881771:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879077.881784:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.881785:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304689705 02000000:00000001:0.0:1788879077.881798:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879077.881804:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.881805:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788879077.881815:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879077.881820:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304689705 02000000:00000001:0.0:1788879077.881821:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879077.881830:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.881832:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879077.881842:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879077.881846:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879077.881851:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.881851:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879077.881866:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879077/real 1788879077] req@ffff94e508978a80 x1875765290127232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879077.881868:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897aa00 x1875765290127488/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788879077.881962:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879077.881965:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879077.881972:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879077.881976:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879077.881979:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.881986:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879077.881993:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879077.881995:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879077.882003:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879077.882006:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290127232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879077.882010:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879077.882021:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879077.882037:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879077.882040:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879077.882042:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879077.882048:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879077.882049:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879077.882059:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879077.882060:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.882066:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:0.0:1788879077.882070:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10016 00000100:00000001:3.0:1788879077.882072:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.882080:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879077.882085:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879077.882087:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879077.882091:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1788879077.882094:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879077.882102:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879077.882107:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000020:00000001:0.0:1788879077.882111:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879077.882114:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:0.0:1788879077.882116:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879077.882123:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:3.0:1788879077.882126:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290127232:192.168.203.129@tcp:250:kworker.0 00000020:00000001:0.0:1788879077.882130:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879077.882138:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879077.882138:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879077.882145:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879077.882146:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879077.882153:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00080000:0.0:1788879077.882158:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:3.0:1788879077.882160:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1788879077.882168:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879077.882170:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:0.0:1788879077.882175:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:3.0:1788879077.882177:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879077.882182:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879077.882183:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879077.882189:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879077.882191:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879077.882195:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879077.882197:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879077.882202:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000008:00100000:0.0:1788879077.882204:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879077.882208:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788879077.882211:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.882214:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879077.882218:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.882227:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1788879077.882229:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879077.882234:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879077.882235:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.882244:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879077.882245:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879077.882251:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879077.882252:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788879077.882256:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.882258:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879077.882264:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879077.882267:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879077.882270:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788879077.882275:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879077.882276:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879077.882287:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879077.882296:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290127360:192.168.203.129@tcp:38:kworker.0 00000100:00100000:0.0:1788879077.882301:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290127616/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879077.882303:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879077.882309:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879077.882312:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879077.882319:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879077.882325:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.882339:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879077.882340:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879077.882344:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00000001:0.0:1788879077.882374:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.882381:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879077.882385:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788879077.882415:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879077.882427:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879077.882435:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879077.882439:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.882443:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879077.882451:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.882460:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304689164 00000100:00000001:3.0:1788879077.882467:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304689705 00000100:00000001:3.0:1788879077.882472:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304689164 00000100:00000001:3.0:1788879077.882478:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304689705 00000100:00000001:3.0:1788879077.882484:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879077.882488:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.882493:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879077.882503:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879077/real 1788879077] req@ffff94e508978700 x1875765290127360/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879077.882519:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879077.882521:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879077.882525:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879077.882527:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879077.882529:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.882532:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879077.882534:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879077.882539:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290127360/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879077.882548:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.882551:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879077.882555:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.882557:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879077.882558:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.882560:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879077.882585:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879077.882590:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879077.882596:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879077.882600:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879077.882607:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290127360:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879077.882615:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879077.882618:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879077.882621:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879077.882626:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879077.882630:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879077.882635:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879077.882641:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290127488:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879077.882648:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879077.882652:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879077.882656:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879077.882658:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879077.882663:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.882672:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879077.882674:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:3.0:1788879077.882680:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879077.882698:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879077.882704:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879077.882707:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879077.882709:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.882711:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879077.882714:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.882719:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304689162 00000100:00000001:3.0:1788879077.882722:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304689705 00000100:00000001:3.0:1788879077.882725:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304689162 00000100:00000001:3.0:1788879077.882728:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304689705 00000100:00000001:3.0:1788879077.882730:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879077.882733:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.882735:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879077.882740:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879077/real 1788879077] req@ffff94e50897aa00 x1875765290127488/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879077.882749:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879077.882751:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879077.882754:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879077.882756:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879077.882758:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.882760:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879077.882762:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879077.882767:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290127488/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879077.882784:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.882787:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879077.882790:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.882792:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879077.882793:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.882795:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879077.882812:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879077.882815:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879077.882818:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879077.882821:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879077.882825:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290127488:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879077.882829:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.882832:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879077.882834:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879077.882836:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879077.882838:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879077.882840:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879077.882842:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879077.882845:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879077.882847:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879077.882850:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879077.882852:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.882854:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879077.882855:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879077.882857:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879077.882859:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879077.882860:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879077.882862:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879077.882864:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879077.882866:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879077.882869:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879077.882870:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.882873:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.882877:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879077.882880:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.882882:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879077.882884:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879077.882886:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879077.882887:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879077.882889:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879077.882892:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879077.882894:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879077.882896:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879077.882900:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290127616:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879077.882941:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879077.882944:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879077.882946:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879077.882948:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879077.882952:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.882959:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879077.882961:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:3.0:1788879077.882966:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879077.882983:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879077.882988:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879077.882991:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879077.882993:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.882995:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879077.882999:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.883003:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304689163 00000100:00000001:3.0:1788879077.883006:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304689705 00000100:00000001:3.0:1788879077.883009:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304689163 00000100:00000001:3.0:1788879077.883012:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304689705 00000100:00000001:3.0:1788879077.883015:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879077.883017:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.883019:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879077.883024:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879077/real 1788879077] req@ffff94e508978a80 x1875765290127616/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879077.883042:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879077.883043:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879077.883046:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879077.883048:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879077.883050:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.883052:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879077.883054:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879077.883058:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290127616/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879077.883066:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.883069:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879077.883072:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.883074:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879077.883076:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.883078:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879077.883094:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879077.883097:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879077.883100:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879077.883102:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879077.883106:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290127616:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879077.883111:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.883113:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879077.883115:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879077.883117:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879077.883119:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879077.883120:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879077.883122:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879077.883124:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879077.883126:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879077.883128:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879077.883130:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.883132:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879077.883136:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879077.883138:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879077.883140:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879077.883142:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879077.945038:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879077.945048:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879077.945052:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879077.945062:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879077.945069:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879077.945075:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879077.945080:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879077.945085:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879077.945090:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879078.649082:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879078.649092:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879078.649099:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879078.649101:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879078.649130:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879078.649134:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879078.649138:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879078.969068:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879078.969076:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879078.969079:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879078.969085:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879078.969089:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879078.969094:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879078.969097:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879078.969099:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879078.969103:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879079.673008:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879079.673016:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879079.673021:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879079.673023:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879079.673046:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879079.673049:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879079.673052:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879079.993021:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879079.993028:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879079.993030:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879079.993035:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879079.993038:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879079.993042:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879079.993045:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879079.993047:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879079.993049:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879080.697138:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879080.697150:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879080.697160:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879080.697163:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879080.697352:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879080.697359:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879080.697365:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879081.017066:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879081.017075:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879081.017078:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879081.017084:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879081.017088:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879081.017092:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879081.017095:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879081.017097:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879081.017100:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879081.721028:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879081.721036:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879081.721043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879081.721044:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879081.721067:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879081.721071:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879081.721074:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879082.041022:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879082.041032:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879082.041035:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879082.041041:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879082.041045:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879082.041050:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879082.041055:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879082.041058:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879082.041061:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879082.744972:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879082.744978:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879082.744981:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879082.744982:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879082.744997:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879082.744999:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879082.745002:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879083.000992:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879083.001000:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879083.001003:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879083.001006:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879083.001027:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879083.001037:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10021 00000100:00080000:2.0:1788879083.001054:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879083.001060:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879083.001063:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879083.001076:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879083.001081:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879083.001086:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879083.001092:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879083.001111:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879083.001114:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879083.001119:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879083.001123:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879083.001127:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879083.001131:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879083.001135:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879083.001137:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879083.001148:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879083.001158:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879083.001159:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879083.001179:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879083.001183:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879083.001189:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879083.001204:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879083.001217:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290127744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879083.001247:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879083.001251:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879083.001262:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879083.001265:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879083.001268:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879083.001269:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879083.001270:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879083.001273:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879083.001279:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10021 00000100:00080000:2.0:1788879083.001282:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879083.001284:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879083.001286:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879083.001288:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879083.001290:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879083.001291:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879083.001293:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879083.001295:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879083.001296:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879083.001299:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879083.001300:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879083.001301:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879083.001305:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879083.001308:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788879083.001312:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879083.001313:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788879083.001328:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001332:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879083.001333:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879083.001337:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788879083.001339:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788879083.001344:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:2.0:1788879083.001345:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879083.001350:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879083.001350:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879083.001359:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879083.001360:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879083.001364:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879083.001367:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290127744:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879083.001368:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879083.001372:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879083.001376:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879083.001377:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879083.001380:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879083.001383:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879083.001385:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879083.001386:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879083.001392:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879083.001394:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290127872/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879083.001402:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879083.001404:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879083.001407:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001409:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788879083.001410:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879083.001412:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000100:00000001:3.0:1788879083.001419:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788879083.001420:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879083.001424:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879083.001426:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879083.001429:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879083.001433:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10021 00000100:00080000:2.0:1788879083.001448:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879083.001451:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879083.001453:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879083.001458:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879083.001460:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879083.001462:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879083.001463:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879083.001468:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304694825 00000020:00000001:2.0:1788879083.001469:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879083.001474:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879083.001475:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879083.001480:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879083.001480:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788879083.001486:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879083.001489:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879083.001492:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879083.001494:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001495:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304694825 00000100:00000001:2.0:1788879083.001496:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879083.001501:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879083.001503:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001503:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879083.001509:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001511:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879083.001515:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879083.001516:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879083.001519:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001519:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879083.001525:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879083.001526:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879083.001528:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879083.001531:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001534:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879083.001537:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001537:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788879083.001539:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879083.001543:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879083.001545:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1788879083.001546:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290128000/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879083.001547:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879083.001550:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879083.001555:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879083.001559:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879083.001560:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879083.001565:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879083.001565:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879083.001573:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879083.001574:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290127872:192.168.203.129@tcp:38:kworker.0 00000100:00080000:2.0:1788879083.001576:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879083.001580:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1788879083.001582:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000001:3.0:1788879083.001583:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788879083.001585:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:3.0:1788879083.001586:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879083.001588:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1788879083.001589:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879083.001593:0:178134:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:3.0:1788879083.001595:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879083.001598:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10021 00000100:00000001:3.0:1788879083.001603:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788879083.001604:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879083.001607:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 00000100:00000001:2.0:1788879083.001608:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879083.001611:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879083.001614:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1788879083.001616:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879083.001618:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879083.001620:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879083.001622:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879083.001624:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879083.001626:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879083.001626:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879083.001630:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879083.001633:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304694283 00000100:00000001:2.0:1788879083.001634:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879083.001639:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879083.001639:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304694825 00000100:00000001:3.0:1788879083.001644:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304694283 02000000:00000001:2.0:1788879083.001645:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879083.001649:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304694825 02000000:00000001:2.0:1788879083.001650:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001655:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879083.001656:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879083.001659:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879083.001660:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788879083.001664:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001666:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879083.001668:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001669:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879083.001674:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879083.001675:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:2.0:1788879083.001676:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879083.001678:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879083.001679:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001681:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879083.001682:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879083.001684:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788879083.001685:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879083.001688:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788879083.001689:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001691:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879083.001694:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879083.001694:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879083.001697:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879083.001700:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879083.001701:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290128128/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879083.001706:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290128000:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879083.001715:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879083.001718:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788879083.001720:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788879083.001721:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879083.001726:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879083.001726:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879083.001732:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879083.001732:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001742:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879083.001745:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:3.0:1788879083.001748:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879083.001755:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001757:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304694281 00000100:00000001:3.0:1788879083.001759:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304694825 00000100:00000001:3.0:1788879083.001761:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304694281 00000100:00000001:3.0:1788879083.001763:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304694825 00000100:00000001:3.0:1788879083.001764:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879083.001765:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001767:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001768:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879083.001770:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879083.001771:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788879083.001772:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879083.001773:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879083.001774:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879083.001774:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879083.001775:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879083.001777:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879083.001778:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879083.001779:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879083.001782:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290128128:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879083.001784:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879083.001785:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879083.001786:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879083.001787:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879083.001789:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001792:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879083.001793:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:3.0:1788879083.001795:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879083.001804:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001807:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304694281 00000100:00000001:3.0:1788879083.001810:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304694825 00000100:00000001:3.0:1788879083.001812:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304694281 00000100:00000001:3.0:1788879083.001815:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304694825 00000100:00000001:3.0:1788879083.001817:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879083.001819:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001822:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001824:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879083.001826:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879083.001828:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788879083.001830:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879083.001831:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879083.001833:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879083.001834:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879083.065004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879083.065010:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879083.065011:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879083.065015:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879083.065018:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879083.065021:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879083.065023:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879083.065025:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879083.065027:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879083.768986:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879083.768991:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879083.768994:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879083.768997:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879083.768998:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879083.769001:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879083.769003:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879083.769004:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879083.769006:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879083.769007:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879083.769022:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879083.769024:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879083.769025:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879084.089012:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879084.089022:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879084.089025:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879084.089032:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879084.089048:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879084.089053:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879084.089056:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879084.089060:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879084.089064:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879084.793044:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879084.793052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879084.793058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879084.793060:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879084.793168:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879084.793176:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879084.793183:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879085.113114:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879085.113125:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879085.113128:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879085.113136:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879085.113141:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879085.113147:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879085.113173:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879085.113178:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879085.113182:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879085.817055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879085.817065:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879085.817068:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879085.817072:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879085.817077:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879085.817080:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879085.817082:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879085.817086:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879085.817090:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879085.817091:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879085.817112:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879085.817115:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879085.817118:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1788879086.073322:0:2350:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:1.0:1788879086.073393:0:2350:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1788879086.073420:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:1.0:1788879086.073441:0:2350:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879086.073472:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879086.073477:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879086.073481:0:2350:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:1.0:1788879086.073497:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:1.0:1788879086.073505:0:2350:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879086.073510:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879086.073513:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879086.073517:0:2350:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:1.0:1788879086.073526:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:1.0:1788879086.073537:0:2350:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879086.073543:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879086.073546:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879086.073550:0:2350:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:1.0:1788879086.073559:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788879086.073566:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879086.073569:0:2350:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879086.073579:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879086.073579:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879086.073589:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879086.073590:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1788879086.073594:0:2350:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:3.0:1788879086.073616:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879083/real 1788879086] req@ffff94e5045e5c00 x1875765290128128/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879086.073647:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879086.073652:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879086.073663:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879086.073668:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879086.073671:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.073678:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879086.073682:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879086.073691:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290128128/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879086.073707:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.073713:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879086.073720:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.073723:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879086.073726:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.073731:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879086.073756:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879086.073764:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879086.073773:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879086.073779:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879086.073787:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290128128:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879086.073796:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879086.073805:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879083/real 1788879086] req@ffff94e5045e5500 x1875765290128000/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879086.073822:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879086.073825:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879086.073830:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879086.073833:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879086.073836:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.073840:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879086.073843:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879086.073851:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290128000/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879086.073865:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.073869:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879086.073874:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.073877:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879086.073880:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.073883:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879086.073948:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879086.073957:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879086.073967:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879086.073973:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879086.073981:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290128000:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879086.073991:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879086.074001:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879083/real 1788879086] req@ffff94e5045e6300 x1875765290127872/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879086.074019:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879086.074023:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879086.074032:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879086.074036:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879086.074039:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.074043:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879086.074046:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879086.074054:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290127872/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879086.074067:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.074072:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879086.074076:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.074079:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879086.074081:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.074084:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879086.074108:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879086.074114:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879086.074120:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879086.074125:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879086.074132:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290127872:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879086.074140:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879086.074149:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879083/real 1788879086] req@ffff94e5045e7800 x1875765290127744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879086.074165:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879086.074167:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879086.074173:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879086.074177:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879086.074180:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.074184:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879086.074187:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879086.074195:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290127744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879086.074208:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.074211:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879086.074215:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.074218:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879086.074220:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.074224:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879086.074227:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879086.074232:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879086.074238:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879086.074242:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879086.074248:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290127744:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879086.074256:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.074260:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879086.074264:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879086.074268:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879086.074276:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879086.074282:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879086.074286:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879086.074330:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879086.074333:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879086.074339:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879086.074342:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.074345:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879086.074347:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879086.074350:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879086.074354:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879086.074356:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879086.074358:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879086.074361:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879086.074363:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879086.074366:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879086.074368:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.074371:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879086.074373:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879086.074376:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879086.074379:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879086.074381:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879086.074383:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879086.074387:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879086.074389:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879086.074392:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879086.074395:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.074398:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879086.074400:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879086.074402:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879086.074405:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879086.074408:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879086.074409:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879086.074413:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879086.074415:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879086.074418:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879086.074420:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.074424:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879086.074437:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879086.074584:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879086.074592:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879086.074596:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879086.137022:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879086.137043:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879086.137046:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879086.137052:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879086.137057:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879086.137061:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879086.137065:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879086.137068:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879086.137073:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879086.841051:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879086.841059:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879086.841065:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879086.841067:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879086.841088:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879086.841092:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879086.841095:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879087.161004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879087.161011:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879087.161013:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879087.161018:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879087.161022:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879087.161025:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879087.161028:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879087.161030:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879087.161032:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879087.865082:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879087.865089:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879087.865094:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879087.865095:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879087.865115:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879087.865118:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879087.865121:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879088.121160:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879088.121181:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879088.121183:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879088.121187:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879088.121191:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879088.121198:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10026 00000100:00080000:0.0:1788879088.121208:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879088.121211:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879088.121214:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879088.121223:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879088.121226:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879088.121241:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879088.121243:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879088.121245:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879088.121248:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879088.121253:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879088.121257:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879088.121260:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879088.121264:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879088.121268:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879088.121270:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879088.121272:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879088.121302:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879088.121306:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879088.121310:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879088.121314:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879088.121331:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879088.121345:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879088.121364:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290128256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879088.121379:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879088.121385:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879088.121416:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879088.121422:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879088.121426:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879088.121428:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879088.121430:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879088.121434:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879088.121437:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10026 00000100:00080000:0.0:1788879088.121443:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879088.121447:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879088.121449:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879088.121454:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879088.121456:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879088.121459:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879088.121462:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879088.121466:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879088.121466:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879088.121469:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879088.121474:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788879088.121477:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879088.121481:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879088.121486:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879088.121491:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879088.121493:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879088.121499:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879088.121500:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879088.121505:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879088.121506:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879088.121512:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879088.121514:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879088.121530:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:3.0:1788879088.121536:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290128256:192.168.203.129@tcp:250:kworker.0 00000008:00100000:0.0:1788879088.121542:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879088.121545:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1788879088.121551:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879088.121554:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788879088.121556:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879088.121561:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879088.121566:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879088.121567:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879088.121571:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879088.121576:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879088.121577:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879088.121579:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879088.121585:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879088.121594:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879088.121602:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290128384/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879088.121615:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879088.121615:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879088.121621:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:0.0:1788879088.121623:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879088.121632:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.121632:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788879088.121640:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879088.121645:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879088.121647:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879088.121649:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879088.121654:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879088.121658:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10026 00000100:00080000:0.0:1788879088.121675:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879088.121679:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879088.121683:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:3.0:1788879088.121685:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:0.0:1788879088.121691:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879088.121695:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879088.121699:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879088.121703:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879088.121703:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879088.121710:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879088.121712:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879088.121713:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879088.121719:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879088.121723:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879088.121725:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879088.121733:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.121736:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879088.121738:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879088.121746:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879088.121747:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304699945 02000000:00000001:0.0:1788879088.121753:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879088.121759:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879088.121762:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879088.121763:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00000001:0.0:1788879088.121767:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879088.121772:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.121775:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304699945 02000000:00000001:0.0:1788879088.121780:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879088.121785:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.121785:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788879088.121793:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879088.121794:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879088.121796:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879088.121800:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.121803:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.121809:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879088.121813:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879088.121821:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879088/real 1788879088] req@ffff94e50897a300 x1875765290128256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879088.121828:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290128512/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788879088.121857:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788879088.121858:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:3.0:1788879088.121862:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879088.121864:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.121872:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879088.121873:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879088.121880:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879088.121880:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879088.121885:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879088.121889:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879088.121892:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879088.121894:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788879088.121898:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879088.121946:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290128256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879088.121949:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879088.121963:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879088.121966:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879088.121971:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10026 00000100:00000001:3.0:1788879088.121975:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879088.121983:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879088.121984:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879088.121991:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879088.121994:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879088.121999:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879088.122001:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879088.122004:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879088.122007:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879088.122010:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879088.122012:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879088.122017:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879088.122021:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879088.122024:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879088.122028:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879088.122031:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879088.122033:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879088.122036:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879088.122039:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879088.122042:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879088.122047:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879088.122049:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879088.122052:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879088.122054:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879088.122057:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879088.122073:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879088.122080:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290128640/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879088.122090:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879088.122093:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879088.122096:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.122187:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879088.122191:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122196:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879088.122200:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879088.122207:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879088.122216:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879088.122220:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879088.122228:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290128256:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879088.122235:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122238:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879088.122242:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879088.122245:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879088.122251:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879088.122253:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879088.122256:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879088.122261:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879088.122264:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879088.122268:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879088.122270:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122273:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122305:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879088.122309:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.122313:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879088.122316:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879088.122318:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879088.122320:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879088.122322:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879088.122326:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879088.122330:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879088.122334:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879088.122340:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290128384:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879088.122346:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879088.122350:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879088.122352:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879088.122355:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879088.122359:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.122368:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879088.122371:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:3.0:1788879088.122377:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879088.122399:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879088.122406:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879088.122409:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879088.122412:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.122414:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879088.122419:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.122424:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304699404 00000100:00000001:3.0:1788879088.122429:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304699945 00000100:00000001:3.0:1788879088.122432:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304699404 00000100:00000001:3.0:1788879088.122433:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304699945 00000100:00000001:3.0:1788879088.122435:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879088.122437:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.122439:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879088.122444:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879088/real 1788879088] req@ffff94e508979c00 x1875765290128384/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879088.122452:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879088.122453:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879088.122456:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879088.122457:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879088.122458:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122460:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879088.122461:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879088.122465:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290128384/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879088.122472:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122474:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879088.122477:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122479:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879088.122481:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122484:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879088.122539:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879088.122546:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879088.122552:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879088.122557:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879088.122564:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290128384:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879088.122571:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879088.122574:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879088.122576:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879088.122596:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879088.122598:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879088.122608:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879088.122613:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290128512:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879088.122617:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879088.122620:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879088.122622:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879088.122624:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879088.122629:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.122636:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879088.122637:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:3.0:1788879088.122641:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879088.122654:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879088.122659:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879088.122661:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879088.122663:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.122665:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879088.122667:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.122670:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304699402 00000100:00000001:3.0:1788879088.122672:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304699945 00000100:00000001:3.0:1788879088.122675:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304699402 00000100:00000001:3.0:1788879088.122677:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304699945 00000100:00000001:3.0:1788879088.122678:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879088.122680:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.122682:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879088.122685:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879088/real 1788879088] req@ffff94e50897ad80 x1875765290128512/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879088.122692:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879088.122693:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879088.122695:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879088.122696:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879088.122697:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122699:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879088.122700:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879088.122703:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290128512/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879088.122709:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122711:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879088.122713:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122725:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879088.122726:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122727:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879088.122737:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879088.122739:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879088.122741:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879088.122743:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879088.122746:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290128512:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879088.122749:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879088.122751:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879088.122752:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879088.122754:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879088.122755:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879088.122757:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879088.122759:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290128640:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879088.122762:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879088.122763:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879088.122764:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879088.122766:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879088.122769:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.122772:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879088.122773:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:3.0:1788879088.122776:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879088.122784:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879088.122787:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879088.122789:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879088.122790:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.122792:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879088.122794:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.122796:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304699402 00000100:00000001:3.0:1788879088.122799:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304699945 00000100:00000001:3.0:1788879088.122800:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304699402 00000100:00000001:3.0:1788879088.122802:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304699945 00000100:00000001:3.0:1788879088.122804:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879088.122806:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.122807:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879088.122810:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879088/real 1788879088] req@ffff94e508978e00 x1875765290128640/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879088.122822:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879088.122823:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879088.122825:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879088.122826:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879088.122827:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122828:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879088.122829:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879088.122832:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290128640/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879088.122837:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122839:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879088.122841:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122842:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879088.122843:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122844:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879088.122854:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879088.122856:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879088.122858:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879088.122860:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879088.122862:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290128640:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879088.122865:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122867:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879088.122868:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879088.122870:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879088.122872:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879088.122873:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879088.122874:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879088.122876:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879088.122878:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879088.122880:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879088.122881:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122883:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879088.122884:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879088.122885:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879088.122887:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879088.122888:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879088.122889:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879088.122890:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879088.122891:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879088.122893:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879088.122895:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122896:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879088.122897:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879088.122897:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879088.122898:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879088.122899:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879088.122923:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879088.122925:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879088.122926:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879088.122927:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879088.122928:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122930:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879088.122933:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879088.122935:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879088.122938:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879088.122940:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879088.185028:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879088.185035:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879088.185037:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879088.185044:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879088.185048:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879088.185053:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879088.185056:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879088.185059:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879088.185063:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879088.889066:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879088.889077:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879088.889086:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879088.889090:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879088.889253:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879088.889261:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879088.889269:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879089.208987:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879089.208993:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879089.208994:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879089.208999:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879089.209002:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879089.209005:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879089.209007:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879089.209009:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879089.209011:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879089.912983:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879089.912989:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879089.912992:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879089.912994:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879089.913009:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879089.913011:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879089.913013:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879090.232972:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879090.232978:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879090.232980:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879090.232984:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879090.232987:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879090.232990:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879090.232992:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879090.232994:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879090.232996:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879090.937007:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879090.937017:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879090.937025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879090.937027:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879090.937067:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879090.937071:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879090.937075:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879091.256976:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879091.256983:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879091.256985:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879091.256990:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879091.256993:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879091.256996:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879091.256998:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879091.257001:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879091.257003:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879091.961159:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879091.961169:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879091.961178:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879091.961180:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879091.961219:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879091.961224:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879091.961229:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879092.281086:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879092.281099:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879092.281102:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879092.281135:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879092.281140:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879092.281146:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879092.281150:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879092.281154:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879092.281158:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879092.985041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879092.985052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879092.985060:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879092.985062:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879092.985201:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879092.985208:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879092.985214:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879093.241169:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879093.241184:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879093.241189:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879093.241195:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879093.241202:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879093.241214:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10032 00000100:00080000:2.0:1788879093.241231:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879093.241236:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879093.241239:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879093.241302:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879093.241305:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879093.241310:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879093.241313:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879093.241315:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879093.241316:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879093.241323:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879093.241326:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879093.241329:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879093.241333:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879093.241337:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879093.241339:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879093.241343:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879093.241377:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879093.241380:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879093.241385:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879093.241388:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879093.241396:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879093.241414:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879093.241428:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5f80 x1875765290128768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879093.241444:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879093.241451:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879093.241473:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879093.241477:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879093.241483:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879093.241485:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879093.241487:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879093.241492:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879093.241496:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10032 00000100:00080000:2.0:1788879093.241503:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879093.241508:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879093.241511:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879093.241519:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879093.241522:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879093.241527:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879093.241532:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879093.241537:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879093.241541:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879093.241545:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788879093.241551:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879093.241554:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879093.241561:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879093.241564:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.241566:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879093.241572:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879093.241575:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879093.241581:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879093.241587:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879093.241590:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879093.241592:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879093.241605:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879093.241610:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290128768:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879093.241613:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879093.241624:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879093.241626:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1788879093.241634:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879093.241635:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788879093.241638:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879093.241642:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879093.241648:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879093.241649:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879093.241654:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879093.241657:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879093.241658:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879093.241681:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879093.241686:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879093.241698:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.241702:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879093.241705:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074556288 : -117763634995328 : ffff94e5045e5f80) 00000100:00100000:2.0:1788879093.241710:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4380 x1875765290128896/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879093.241712:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879093.241728:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879093.241733:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879093.241739:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879093.241743:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879093.241749:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879093.241752:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879093.241753:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879093.241757:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879093.241760:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10032 00000100:00080000:2.0:1788879093.241766:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1788879093.241771:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879093.241772:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879093.241777:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879093.241781:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879093.241784:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879093.241787:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879093.241789:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879093.241792:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879093.241793:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879093.241798:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879093.241801:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788879093.241805:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879093.241807:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879093.241811:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.241812:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:2.0:1788879093.241817:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879093.241821:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879093.241821:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879093.241827:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.241828:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304705065 00000100:00000001:2.0:1788879093.241830:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879093.241835:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879093.241837:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879093.241839:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879093.241845:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879093.241845:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1788879093.241847:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879093.241853:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879093.241854:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879093.241856:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879093.241863:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879093.241868:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290129024/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879093.241869:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304705065 00000100:00000001:3.0:1788879093.241880:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879093.241882:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879093.241886:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.241887:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879093.241890:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.241893:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879093.241894:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879093.241899:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879093.241940:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879093/real 1788879093] req@ffff94e5045e5f80 x1875765290128768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879093.241963:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879093.241970:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879093.241977:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879093.241985:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000400:00000001:3.0:1788879093.241985:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879093.241992:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1788879093.241998:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10032 00000100:00000001:3.0:1788879093.242002:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:2.0:1788879093.242007:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879093.242009:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879093.242014:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879093.242016:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879093.242020:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879093.242022:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879093.242026:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788879093.242030:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879093.242036:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:3.0:1788879093.242039:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5f80 x1875765290128768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788879093.242041:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879093.242052:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879093.242063:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879093.242066:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879093.242068:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879093.242075:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879093.242080:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879093.242083:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879093.242090:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242091:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879093.242095:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879093.242098:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879093.242101:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1788879093.242106:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879093.242107:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879093.242119:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1788879093.242120:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879093.242130:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00080000:3.0:1788879093.242135:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000008:00100000:2.0:1788879093.242136:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879093.242141:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000001:2.0:1788879093.242144:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879093.242152:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879093.242153:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290128768:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879093.242162:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879093.242164:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879093.242167:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242170:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879093.242174:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879093.242176:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879093.242181:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879093.242183:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879093.242188:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879093.242190:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879093.242196:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879093.242199:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:2.0:1788879093.242200:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879093.242222:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879093.242226:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00100000:2.0:1788879093.242228:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290129152/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879093.242231:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879093.242234:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242241:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879093.242263:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879093.242269:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879093.242273:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242275:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879093.242279:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242282:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879093.242285:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879093.242286:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879093.242288:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879093.242289:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879093.242293:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879093.242295:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879093.242298:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879093.242303:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290128896:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879093.242306:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879093.242308:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879093.242311:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879093.242312:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879093.242315:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242322:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879093.242324:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074549120 : -117763635002496 : ffff94e5045e4380) 00000100:00000001:3.0:1788879093.242329:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879093.242342:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879093.242347:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879093.242349:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879093.242350:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242352:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879093.242354:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242359:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304704524 00000100:00000001:3.0:1788879093.242362:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304705065 00000100:00000001:3.0:1788879093.242364:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304704524 00000100:00000001:3.0:1788879093.242366:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304705065 00000100:00000001:3.0:1788879093.242369:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879093.242370:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242372:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879093.242376:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879093/real 1788879093] req@ffff94e5045e4380 x1875765290128896/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879093.242383:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879093.242385:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879093.242387:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879093.242390:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879093.242391:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242394:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879093.242396:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879093.242402:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4380 x1875765290128896/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879093.242412:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242415:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879093.242420:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242423:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879093.242425:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242427:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879093.242449:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879093.242454:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879093.242460:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879093.242462:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879093.242465:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290128896:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879093.242469:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879093.242470:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879093.242472:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879093.242474:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879093.242476:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879093.242478:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879093.242480:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290129024:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879093.242483:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879093.242485:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879093.242487:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879093.242489:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879093.242492:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242499:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879093.242501:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000100:00000001:3.0:1788879093.242505:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879093.242517:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879093.242521:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879093.242524:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879093.242525:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242527:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879093.242529:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242533:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304704522 00000100:00000001:3.0:1788879093.242535:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304705065 00000100:00000001:3.0:1788879093.242538:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304704522 00000100:00000001:3.0:1788879093.242540:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304705065 00000100:00000001:3.0:1788879093.242543:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879093.242545:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242546:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879093.242550:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879093/real 1788879093] req@ffff94e5045e7800 x1875765290129024/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879093.242556:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879093.242557:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879093.242560:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879093.242561:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879093.242563:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242564:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879093.242566:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879093.242569:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290129024/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879093.242575:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242576:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879093.242578:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242580:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879093.242581:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242582:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879093.242597:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879093.242599:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879093.242601:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879093.242604:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879093.242607:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290129024:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879093.242610:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879093.242611:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879093.242613:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879093.242615:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879093.242617:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879093.242619:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879093.242621:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290129152:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879093.242623:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879093.242625:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879093.242626:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879093.242627:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879093.242630:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242633:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879093.242634:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 00000100:00000001:3.0:1788879093.242637:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879093.242645:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879093.242648:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879093.242650:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879093.242652:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242653:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879093.242655:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242658:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304704522 00000100:00000001:3.0:1788879093.242660:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304705065 00000100:00000001:3.0:1788879093.242663:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304704522 00000100:00000001:3.0:1788879093.242665:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304705065 00000100:00000001:3.0:1788879093.242667:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879093.242668:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242670:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879093.242673:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879093/real 1788879093] req@ffff94e5045e6300 x1875765290129152/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879093.242679:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879093.242680:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879093.242682:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879093.242683:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879093.242684:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242686:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879093.242687:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879093.242690:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290129152/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879093.242697:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242699:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879093.242701:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242702:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879093.242703:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242704:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879093.242718:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879093.242720:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879093.242723:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879093.242724:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879093.242727:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290129152:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879093.242730:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242731:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879093.242749:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879093.242752:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879093.242754:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879093.242756:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879093.242758:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879093.242761:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879093.242763:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879093.242766:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879093.242767:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242769:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879093.242771:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879093.242772:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879093.242774:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879093.242775:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879093.242777:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879093.242780:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879093.242781:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879093.242784:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879093.242786:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242788:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879093.242790:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879093.242791:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879093.242794:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879093.242795:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879093.242797:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879093.242799:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879093.242800:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879093.242802:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879093.242803:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242805:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879093.242809:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879093.242811:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879093.242813:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879093.242815:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879093.304998:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879093.305007:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879093.305009:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879093.305015:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879093.305019:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879093.305025:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879093.305028:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879093.305031:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879093.305034:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879094.009100:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879094.009109:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879094.009116:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879094.009120:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879094.009124:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879094.009128:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879094.009131:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879094.009134:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879094.009136:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879094.009139:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879094.009175:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879094.009179:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879094.009183:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879094.329039:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879094.329049:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879094.329052:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879094.329059:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879094.329064:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879094.329069:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879094.329072:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879094.329076:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879094.329079:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879095.033013:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879095.033022:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879095.033024:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879095.033028:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879095.033033:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879095.033035:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879095.033037:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879095.033038:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879095.033042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879095.033043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879095.033061:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879095.033064:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879095.033066:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879095.352981:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879095.352987:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879095.352989:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879095.352994:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879095.352998:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879095.353002:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879095.353004:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879095.353007:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879095.353009:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879096.056974:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879096.056981:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879096.056986:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879096.056987:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879096.057004:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879096.057006:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879096.057009:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879096.377054:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879096.377064:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879096.377066:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879096.377072:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879096.377076:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879096.377081:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879096.377084:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879096.377088:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879096.377090:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879097.081027:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879097.081039:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879097.081043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879097.081044:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879097.081121:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879097.081126:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879097.081130:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879097.401030:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879097.401039:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879097.401041:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879097.401047:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879097.401061:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879097.401065:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879097.401068:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879097.401071:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879097.401074:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879098.105048:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879098.105056:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879098.105063:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879098.105064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879098.105088:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879098.105092:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879098.105095:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879098.361048:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879098.361063:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879098.361067:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879098.361072:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879098.361081:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879098.361091:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10037 00000100:00080000:2.0:1788879098.361109:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879098.361113:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879098.361117:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879098.361129:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879098.361134:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879098.361140:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879098.361144:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879098.361147:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879098.361149:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879098.361168:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879098.361171:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879098.361176:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879098.361181:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879098.361186:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879098.361189:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879098.361192:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879098.361228:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879098.361231:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879098.361235:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879098.361240:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879098.361247:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879098.361265:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879098.361279:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290129280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879098.361294:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879098.361300:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879098.361318:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879098.361323:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879098.361328:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879098.361330:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879098.361333:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879098.361337:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879098.361340:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10037 00000100:00080000:2.0:1788879098.361346:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879098.361349:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879098.361352:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879098.361359:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879098.361360:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879098.361364:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879098.361367:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879098.361371:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879098.361373:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879098.361375:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879098.361384:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879098.361384:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879098.361391:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.361391:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788879098.361399:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879098.361401:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879098.361413:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879098.361419:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879098.361421:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879098.361428:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879098.361430:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879098.361441:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788879098.361443:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788879098.361454:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879098.361459:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290129280:192.168.203.129@tcp:250:kworker.0 00000008:00000001:2.0:1788879098.361463:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879098.361474:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.361477:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879098.361482:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879098.361484:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879098.361486:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879098.361488:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879098.361491:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879098.361495:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879098.361496:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879098.361511:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879098.361515:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879098.361521:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879098.361533:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290129408/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879098.361536:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879098.361541:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:2.0:1788879098.361557:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879098.361558:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879098.361562:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879098.361568:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879098.361572:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879098.361576:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879098.361579:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879098.361582:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879098.361585:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879098.361589:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10037 00000100:00080000:2.0:1788879098.361593:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879098.361596:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879098.361598:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879098.361601:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879098.361603:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879098.361605:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000800:00000100:3.0:1788879098.361615:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:2.0:1788879098.361616:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879098.361621:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879098.361623:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879098.361631:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879098.361637:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879098.361641:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879098.361650:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879098.361650:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879098.361662:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879098.361665:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879098.361665:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879098.361674:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879098.361676:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304710185 02000000:00000001:2.0:1788879098.361681:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879098.361685:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879098.361688:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879098.361690:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00000001:2.0:1788879098.361694:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879098.361698:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.361701:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304710185 02000000:00000001:2.0:1788879098.361703:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879098.361707:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.361709:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879098.361711:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879098.361715:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879098.361716:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879098.361724:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.361725:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.361747:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879098.361752:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879098.361761:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879098/real 1788879098] req@ffff94e5045e5500 x1875765290129280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879098.361776:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290129536/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788879098.361785:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879098.361790:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879098.361798:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879098.361805:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879098.361815:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879098.361815:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879098.361825:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.361826:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879098.361831:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879098.361834:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879098.361840:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1788879098.361842:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290129280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879098.361857:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879098.361866:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788879098.361868:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879098.361870:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879098.361877:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879098.361877:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879098.361894:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10037 00000100:00000001:3.0:1788879098.361895:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.361941:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879098.361943:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.361946:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879098.361950:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879098.361951:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879098.361962:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00080000:3.0:1788879098.361963:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788879098.361970:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879098.361981:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000020:00000001:2.0:1788879098.361982:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879098.361985:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879098.361990:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879098.361992:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:2.0:1788879098.361995:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879098.362005:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879098.362008:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:3.0:1788879098.362011:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290129280:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1788879098.362016:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879098.362025:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879098.362029:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879098.362033:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879098.362035:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879098.362038:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788879098.362040:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362042:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788879098.362046:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788879098.362049:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000008:00100000:2.0:1788879098.362051:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788879098.362055:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000008:00000001:2.0:1788879098.362057:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879098.362060:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:2.0:1788879098.362061:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879098.362064:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879098.362073:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879098.362075:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879098.362085:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879098.362087:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879098.362096:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879098.362097:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 02000000:00000001:2.0:1788879098.362099:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879098.362100:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879098.362112:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362113:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879098.362140:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362141:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879098.362148:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362155:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1788879098.362158:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290129664/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879098.362158:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879098.362164:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879098.362170:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879098.362174:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879098.362184:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879098.362191:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879098.362193:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788879098.362199:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879098.362200:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879098.362230:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290129408:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879098.362235:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362237:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879098.362247:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879098.362250:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879098.362253:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879098.362257:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362264:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879098.362265:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:3.0:1788879098.362269:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879098.362287:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879098.362290:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879098.362292:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879098.362294:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362295:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879098.362297:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362300:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304709644 00000100:00000001:3.0:1788879098.362302:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304710185 00000100:00000001:3.0:1788879098.362304:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304709644 00000100:00000001:3.0:1788879098.362306:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304710185 00000100:00000001:3.0:1788879098.362307:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879098.362308:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362309:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879098.362312:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879098/real 1788879098] req@ffff94e5045e5c00 x1875765290129408/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879098.362318:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879098.362319:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879098.362320:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879098.362321:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879098.362322:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362324:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879098.362325:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879098.362327:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290129408/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879098.362332:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362333:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879098.362334:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362335:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879098.362336:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362337:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879098.362348:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879098.362350:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879098.362353:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879098.362355:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879098.362357:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290129408:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879098.362360:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879098.362361:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879098.362362:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879098.362364:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879098.362365:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879098.362366:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879098.362368:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290129536:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879098.362370:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879098.362372:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879098.362373:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879098.362374:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879098.362376:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362379:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879098.362380:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:3.0:1788879098.362382:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879098.362388:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879098.362391:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879098.362392:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879098.362393:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362394:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879098.362395:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362398:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304709642 00000100:00000001:3.0:1788879098.362400:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304710185 00000100:00000001:3.0:1788879098.362401:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304709642 00000100:00000001:3.0:1788879098.362403:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304710185 00000100:00000001:3.0:1788879098.362404:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879098.362405:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362406:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879098.362409:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879098/real 1788879098] req@ffff94e5045e7100 x1875765290129536/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879098.362418:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879098.362419:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879098.362421:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879098.362421:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879098.362422:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362423:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879098.362424:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879098.362426:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290129536/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879098.362430:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362431:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879098.362433:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362434:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879098.362434:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362436:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879098.362444:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879098.362446:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879098.362448:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879098.362450:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879098.362452:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290129536:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879098.362454:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362455:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879098.362456:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879098.362457:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879098.362458:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879098.362459:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879098.362460:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879098.362462:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879098.362463:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879098.362464:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879098.362465:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362466:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879098.362467:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879098.362468:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879098.362469:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879098.362470:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879098.362470:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879098.362472:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879098.362472:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879098.362473:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879098.362474:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362475:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362478:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879098.362478:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362480:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879098.362481:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879098.362482:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879098.362483:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879098.362484:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879098.362485:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879098.362486:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879098.362488:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879098.362490:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290129664:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879098.362492:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879098.362493:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879098.362494:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879098.362495:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879098.362497:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362499:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879098.362500:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:3.0:1788879098.362502:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879098.362508:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879098.362510:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879098.362511:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879098.362512:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362513:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879098.362514:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362517:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304709642 00000100:00000001:3.0:1788879098.362519:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304710185 00000100:00000001:3.0:1788879098.362520:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304709642 00000100:00000001:3.0:1788879098.362522:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304710185 00000100:00000001:3.0:1788879098.362524:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879098.362525:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362526:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879098.362528:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879098/real 1788879098] req@ffff94e5045e4a80 x1875765290129664/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879098.362533:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879098.362534:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879098.362537:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879098.362538:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879098.362539:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362541:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879098.362543:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879098.362546:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290129664/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879098.362553:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362555:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879098.362557:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362559:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879098.362560:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362562:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879098.362575:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879098.362578:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879098.362582:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879098.362584:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879098.362588:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290129664:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879098.362590:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362592:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879098.362593:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879098.362594:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879098.362596:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879098.362597:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879098.362598:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879098.362600:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879098.362601:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879098.362602:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879098.362603:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362605:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879098.362607:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879098.362608:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879098.362609:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879098.362610:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879098.425006:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879098.425012:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879098.425014:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879098.425019:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879098.425022:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879098.425025:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879098.425028:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879098.425030:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879098.425034:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879099.129126:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879099.129135:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879099.129144:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879099.129146:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879099.129187:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879099.129191:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879099.129196:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879099.449067:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879099.449076:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879099.449078:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879099.449085:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879099.449090:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879099.449095:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879099.449099:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879099.449102:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879099.449106:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879100.153067:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879100.153086:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879100.153091:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879100.153093:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879100.153117:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879100.153120:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879100.153123:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879100.473161:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879100.473176:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879100.473180:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879100.473234:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879100.473241:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879100.473248:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879100.473253:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879100.473258:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879100.473264:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879101.177017:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879101.177024:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879101.177029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879101.177031:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879101.177074:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879101.177079:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879101.177084:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879101.497085:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879101.497094:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879101.497097:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879101.497104:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879101.497108:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879101.497113:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879101.497117:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879101.497121:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879101.497126:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879102.201069:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879102.201077:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879102.201084:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879102.201086:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879102.201110:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879102.201114:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879102.201117:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879102.521002:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879102.521010:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879102.521013:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879102.521018:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879102.521023:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879102.521028:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879102.521031:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879102.521034:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879102.521037:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879103.225071:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879103.225080:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879103.225086:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879103.225091:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879103.225094:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879103.225099:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879103.225102:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879103.225105:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879103.225107:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879103.225110:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879103.225148:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879103.225152:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879103.225155:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879103.481033:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879103.481042:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879103.481044:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879103.481055:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879103.481060:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879103.481066:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10042 00000100:00080000:3.0:1788879103.481076:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879103.481078:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879103.481080:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879103.481086:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879103.481088:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879103.481091:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879103.481093:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879103.481095:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879103.481096:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879103.481099:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879103.481101:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879103.481104:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879103.481107:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879103.481110:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879103.481111:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879103.481113:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879103.481125:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879103.481126:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879103.481128:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879103.481131:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879103.481135:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879103.481148:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879103.481158:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290129792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879103.481171:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879103.481190:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879103.481213:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879103.481216:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879103.481219:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879103.481220:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879103.481222:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879103.481224:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879103.481227:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10042 00000100:00080000:3.0:1788879103.481230:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879103.481232:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879103.481234:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879103.481237:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879103.481238:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879103.481240:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879103.481242:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879103.481244:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879103.481245:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879103.481249:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879103.481252:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879103.481253:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879103.481258:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788879103.481259:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879103.481264:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481265:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879103.481266:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879103.481271:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1788879103.481272:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879103.481275:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:3.0:1788879103.481276:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481278:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879103.481281:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879103.481284:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879103.481288:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879103.481289:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879103.481291:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879103.481294:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879103.481299:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290129792:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879103.481300:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879103.481304:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879103.481307:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879103.481310:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879103.481312:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879103.481316:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879103.481318:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879103.481320:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879103.481323:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879103.481331:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290129920/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879103.481340:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879103.481344:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:3.0:1788879103.481344:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879103.481350:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481351:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879103.481356:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879103.481359:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879103.481362:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879103.481364:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879103.481365:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879103.481368:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879103.481370:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10042 00000100:00080000:3.0:1788879103.481374:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879103.481376:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879103.481377:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879103.481380:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879103.481381:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879103.481383:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000800:00000100:1.0:1788879103.481388:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:3.0:1788879103.481389:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879103.481394:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879103.481396:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879103.481399:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879103.481403:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879103.481403:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879103.481407:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879103.481410:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788879103.481411:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879103.481415:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879103.481415:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481420:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879103.481421:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879103.481425:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879103.481426:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788879103.481428:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879103.481431:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481432:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304715305 02000000:00000001:3.0:1788879103.481437:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879103.481439:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879103.481441:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879103.481442:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879103.481444:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788879103.481444:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879103.481454:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879103.481458:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290130048/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879103.481463:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879103.481464:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304715305 00000100:00000001:3.0:1788879103.481465:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879103.481472:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481475:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788879103.481475:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879103.481482:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879103.481482:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879103.481487:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879103.481488:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879103.481490:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879103.481493:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879103.481494:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879103.481498:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10042 00000100:00100000:1.0:1788879103.481504:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879103/real 1788879103] req@ffff94e50449c000 x1875765290129792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879103.481504:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879103.481509:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879103.481512:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879103.481519:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879103.481522:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000001:1.0:1788879103.481524:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1788879103.481524:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000400:00000001:1.0:1788879103.481528:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1788879103.481529:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879103.481532:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879103.481533:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879103.481534:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879103.481538:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00080000:3.0:1788879103.481541:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879103.481542:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879103.481546:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481547:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788879103.481550:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879103.481551:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879103.481555:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788879103.481560:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290129792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879103.481561:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879103.481565:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879103.481568:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879103.481574:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879103.481578:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879103.481584:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879103.481587:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879103.481592:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879103.481596:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879103.481596:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879103.481600:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481602:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879103.481605:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481606:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.481610:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879103.481612:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879103.481613:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290130176/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879103.481614:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879103.481618:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879103.481625:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788879103.481630:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00080000:1.0:1788879103.481634:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879103.481635:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481641:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788879103.481642:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879103.481651:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290129792:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879103.481656:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.481658:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879103.481659:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879103.481661:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879103.481664:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879103.481666:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879103.481667:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879103.481671:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879103.481672:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879103.481674:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879103.481675:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.481677:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.481683:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879103.481685:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481687:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879103.481688:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879103.481690:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879103.481691:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879103.481692:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879103.481694:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879103.481695:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879103.481697:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879103.481699:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290129920:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879103.481702:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879103.481703:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879103.481705:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879103.481706:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879103.481708:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481713:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879103.481714:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:1.0:1788879103.481718:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879103.481726:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879103.481730:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879103.481732:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879103.481733:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481734:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879103.481736:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481739:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304714763 00000100:00000001:1.0:1788879103.481741:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304715305 00000100:00000001:1.0:1788879103.481743:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304714763 00000100:00000001:1.0:1788879103.481745:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304715305 00000100:00000001:1.0:1788879103.481746:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879103.481748:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481749:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879103.481752:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879103/real 1788879103] req@ffff94e50449ca80 x1875765290129920/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879103.481758:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879103.481759:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879103.481760:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879103.481761:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879103.481763:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.481764:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879103.481765:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879103.481768:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290129920/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879103.481778:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.481779:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879103.481781:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.481782:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879103.481783:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.481784:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879103.481794:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879103.481796:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879103.481798:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879103.481800:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879103.481802:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290129920:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879103.481805:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879103.481806:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879103.481807:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879103.481808:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879103.481809:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879103.481811:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879103.481813:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290130048:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879103.481815:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879103.481816:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879103.481817:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879103.481818:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879103.481820:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481823:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879103.481824:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:1.0:1788879103.481827:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879103.481834:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879103.481836:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879103.481838:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879103.481839:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481840:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879103.481842:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481844:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304714761 00000100:00000001:1.0:1788879103.481846:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304715305 00000100:00000001:1.0:1788879103.481847:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304714761 00000100:00000001:1.0:1788879103.481849:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304715305 00000100:00000001:1.0:1788879103.481850:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879103.481852:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481853:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879103.481855:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879103/real 1788879103] req@ffff94e50449f100 x1875765290130048/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879103.481861:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879103.481861:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879103.481863:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879103.481864:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879103.481865:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.481866:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879103.481867:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879103.481869:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290130048/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879103.481874:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.481876:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879103.481877:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.481879:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879103.481880:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.481881:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879103.481889:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879103.481891:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879103.481892:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879103.481894:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879103.481897:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290130048:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879103.481899:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879103.481901:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879103.481938:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879103.481941:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879103.481942:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879103.481943:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879103.481946:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290130176:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879103.481948:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879103.481949:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879103.481950:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879103.481952:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879103.481954:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481958:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879103.481959:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:1.0:1788879103.481961:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879103.481969:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879103.481972:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879103.481974:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879103.481975:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481976:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879103.481978:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481980:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304714762 00000100:00000001:1.0:1788879103.481982:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304715305 00000100:00000001:1.0:1788879103.481984:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304714762 00000100:00000001:1.0:1788879103.481986:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304715305 00000100:00000001:1.0:1788879103.481987:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879103.481988:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.481990:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879103.481992:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879103/real 1788879103] req@ffff94e50449e300 x1875765290130176/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879103.482002:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879103.482003:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879103.482005:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879103.482006:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879103.482007:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.482008:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879103.482009:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879103.482012:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290130176/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879103.482017:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.482018:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879103.482020:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.482021:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879103.482022:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.482024:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879103.482032:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879103.482034:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879103.482036:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879103.482038:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879103.482040:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290130176:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879103.482042:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.482043:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879103.482044:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879103.482045:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879103.482047:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879103.482048:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879103.482049:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879103.482050:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879103.482051:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879103.482053:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879103.482054:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.482055:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879103.482070:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879103.482071:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879103.482073:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879103.482074:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879103.482074:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879103.482076:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879103.482077:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879103.482078:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879103.482079:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.482080:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879103.482081:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879103.482082:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879103.482083:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879103.482083:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879103.482084:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879103.482085:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879103.482086:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879103.482087:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879103.482088:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.482090:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.482092:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879103.482093:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.482095:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879103.482096:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879103.545014:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879103.545020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879103.545022:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879103.545028:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879103.545031:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879103.545034:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.545037:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879103.545040:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879103.545042:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879104.249032:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879104.249040:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879104.249047:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879104.249049:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879104.249092:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879104.249095:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879104.249100:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879104.569066:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879104.569074:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879104.569076:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879104.569082:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879104.569086:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879104.569108:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879104.569111:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879104.569114:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879104.569117:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879105.273027:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879105.273036:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879105.273058:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879105.273062:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879105.273068:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879105.273071:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879105.273074:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879105.273077:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879105.273082:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879105.273084:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879105.273157:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879105.273180:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879105.273187:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879105.593029:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879105.593041:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879105.593044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879105.593064:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879105.593070:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879105.593075:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879105.593079:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879105.593083:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879105.593087:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879106.297119:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879106.297128:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879106.297134:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879106.297136:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879106.297253:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879106.297259:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879106.297266:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879106.616983:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879106.616991:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879106.616994:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879106.616999:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879106.617003:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879106.617007:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879106.617010:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879106.617012:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879106.617015:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879107.321084:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879107.321092:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879107.321098:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879107.321099:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879107.321126:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879107.321129:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879107.321132:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879107.641043:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879107.641052:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879107.641054:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879107.641061:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879107.641065:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879107.641070:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879107.641072:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879107.641076:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879107.641079:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879108.345067:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879108.345076:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879108.345084:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879108.345085:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879108.345131:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879108.345135:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879108.345139:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879108.601106:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879108.601117:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879108.601120:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879108.601126:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879108.601131:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879108.601162:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10047 00000100:00080000:3.0:1788879108.601176:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879108.601179:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879108.601182:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879108.601191:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879108.601194:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879108.601198:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879108.601201:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879108.601204:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879108.601205:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879108.601211:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879108.601215:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879108.601219:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879108.601223:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879108.601228:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879108.601230:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879108.601233:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879108.601253:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879108.601255:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879108.601258:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879108.601261:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879108.601266:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879108.601279:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879108.601289:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290130304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879108.601300:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879108.601305:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879108.601319:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879108.601323:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879108.601327:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879108.601329:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879108.601332:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879108.601337:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879108.601342:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10047 00000100:00080000:3.0:1788879108.601346:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879108.601352:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879108.601354:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879108.601362:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879108.601364:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879108.601372:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788879108.601372:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879108.601385:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1788879108.601385:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879108.601396:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879108.601397:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879108.601411:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:1.0:1788879108.601413:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879108.601415:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879108.601418:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879108.601419:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879108.601427:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879108.601427:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879108.601436:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879108.601440:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1788879108.601443:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290130304:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788879108.601447:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879108.601456:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879108.601457:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879108.601464:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879108.601467:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:3.0:1788879108.601469:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879108.601471:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788879108.601479:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879108.601480:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879108.601486:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879108.601495:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879108.601496:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879108.601503:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879108.601507:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879108.601509:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879108.601529:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.601534:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879108.601537:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.601538:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:1.0:1788879108.601547:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1788879108.601555:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290130432/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879108.601573:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879108.601576:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879108.601586:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879108.601593:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788879108.601602:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1788879108.601604:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879108.601610:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879108.601614:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879108.601622:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879108.601623:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879108.601633:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1788879108.601637:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10047 00000100:00000001:1.0:1788879108.601638:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.601642:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879108.601659:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.601667:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304720425 00000100:00080000:3.0:1788879108.601668:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879108.601673:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879108.601676:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879108.601680:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1788879108.601683:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879108.601686:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879108.601690:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879108.601693:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879108.601695:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879108.601697:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879108.601701:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879108.601704:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.601706:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304720425 02000000:00000001:3.0:1788879108.601708:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879108.601722:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1788879108.601723:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879108.601735:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.601737:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879108.601740:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879108.601742:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788879108.601746:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879108.601748:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000008:00000001:3.0:1788879108.601759:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879108.601767:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879108/real 1788879108] req@ffff94e50449c700 x1875765290130304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879108.601769:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879108.601788:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879108.601792:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879108.601793:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788879108.601796:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:1.0:1788879108.601800:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788879108.601800:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879108.601812:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.601815:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879108.601819:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.601823:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879108.601830:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879108.601831:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290130560/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879108.601846:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879108.601854:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879108.601855:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879108.601865:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879108.601870:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879108.601871:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290130304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879108.601877:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879108.601899:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879108.601903:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879108.601906:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879108.601911:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879108.601916:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10047 00000100:00080000:3.0:1788879108.601922:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879108.601924:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879108.601926:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879108.601930:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879108.601932:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879108.601939:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.601944:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879108.601950:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.601953:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879108.601954:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.601958:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879108.601960:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879108.601963:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879108.601969:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879108.601971:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879108.601978:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290130304:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879108.601985:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788879108.601987:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879108.601990:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879108.601997:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1788879108.601999:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788879108.602002:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788879108.602005:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879108.602009:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1788879108.602009:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879108.602018:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00080000:3.0:1788879108.602022:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000020:00000001:1.0:1788879108.602023:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788879108.602031:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879108.602033:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879108.602036:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:3.0:1788879108.602036:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879108.602062:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879108.602065:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879108.602067:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879108.602070:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879108.602072:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879108.602076:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879108.602079:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879108.602081:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602084:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879108.602084:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879108.602090:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879108.602091:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879108.602094:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879108.602098:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879108.602100:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788879108.602100:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879108.602108:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602109:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879108.602115:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879108.602116:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602118:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879108.602121:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879108.602128:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290130688/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879108.602130:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879108.602136:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879108.602153:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879108.602158:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290130432:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879108.602173:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879108.602178:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879108.602181:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879108.602183:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879108.602184:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879108.602188:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879108.602189:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879108.602195:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602200:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879108.602203:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:1.0:1788879108.602208:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879108.602224:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879108.602229:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879108.602235:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879108.602238:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602239:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879108.602243:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602247:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304719884 00000100:00000001:1.0:1788879108.602249:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304720425 00000100:00000001:1.0:1788879108.602251:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304719884 00000100:00000001:1.0:1788879108.602253:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304720425 00000100:00000001:1.0:1788879108.602255:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879108.602256:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602258:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879108.602261:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879108/real 1788879108] req@ffff94e50449f480 x1875765290130432/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879108.602267:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879108.602268:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879108.602270:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879108.602272:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879108.602273:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602274:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879108.602275:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879108.602278:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290130432/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879108.602284:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602285:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879108.602288:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602289:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879108.602290:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602291:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879108.602307:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879108.602309:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879108.602312:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879108.602313:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879108.602316:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290130432:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879108.602318:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879108.602319:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879108.602320:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879108.602322:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879108.602324:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879108.602326:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879108.602328:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290130560:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879108.602330:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879108.602332:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879108.602333:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879108.602334:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879108.602337:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602341:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879108.602341:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:1.0:1788879108.602344:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879108.602351:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879108.602354:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879108.602355:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879108.602356:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602358:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879108.602360:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602362:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304719882 00000100:00000001:1.0:1788879108.602364:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304720425 00000100:00000001:1.0:1788879108.602366:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304719882 00000100:00000001:1.0:1788879108.602368:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304720425 00000100:00000001:1.0:1788879108.602370:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879108.602371:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602372:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879108.602375:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879108/real 1788879108] req@ffff94e50449f800 x1875765290130560/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879108.602380:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879108.602380:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879108.602382:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879108.602383:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879108.602383:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602385:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879108.602385:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879108.602388:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290130560/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879108.602393:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602394:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879108.602396:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602397:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879108.602398:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602399:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879108.602412:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879108.602413:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879108.602415:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879108.602417:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879108.602419:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290130560:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879108.602422:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602424:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879108.602425:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879108.602426:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879108.602428:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879108.602429:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879108.602430:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879108.602432:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879108.602433:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879108.602435:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879108.602436:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602437:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879108.602438:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879108.602439:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879108.602441:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879108.602442:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879108.602442:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879108.602444:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879108.602444:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879108.602445:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879108.602446:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602447:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602449:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879108.602450:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602452:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879108.602453:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879108.602454:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879108.602455:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879108.602456:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879108.602457:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879108.602459:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879108.602460:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879108.602462:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290130688:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879108.602464:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879108.602466:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879108.602466:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879108.602468:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879108.602469:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602472:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879108.602473:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:1.0:1788879108.602476:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879108.602483:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879108.602485:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879108.602487:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879108.602488:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602489:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879108.602491:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602494:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304719882 00000100:00000001:1.0:1788879108.602496:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304720425 00000100:00000001:1.0:1788879108.602497:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304719882 00000100:00000001:1.0:1788879108.602499:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304720425 00000100:00000001:1.0:1788879108.602501:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879108.602502:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602503:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879108.602505:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879108/real 1788879108] req@ffff94e50449d180 x1875765290130688/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879108.602510:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879108.602511:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879108.602519:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879108.602521:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879108.602522:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602523:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879108.602524:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879108.602526:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290130688/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879108.602531:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602533:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879108.602534:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602535:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879108.602536:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602537:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879108.602550:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879108.602551:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879108.602554:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879108.602556:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879108.602558:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290130688:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879108.602561:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602562:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879108.602563:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879108.602564:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879108.602565:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879108.602565:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879108.602566:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879108.602568:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879108.602568:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879108.602569:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879108.602570:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602571:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.602573:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879108.602574:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.602576:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879108.602577:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879108.665021:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879108.665031:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879108.665033:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879108.665040:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879108.665044:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879108.665049:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.665052:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879108.665056:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879108.665059:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879109.369057:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879109.369067:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879109.369075:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879109.369077:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879109.369194:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879109.369200:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879109.369206:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879109.689027:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879109.689038:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879109.689040:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879109.689048:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879109.689053:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879109.689076:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879109.689080:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879109.689083:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879109.689088:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879110.393029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879110.393038:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879110.393044:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879110.393045:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879110.393115:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879110.393119:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879110.393123:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879110.713074:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879110.713082:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879110.713084:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879110.713091:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879110.713094:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879110.713098:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879110.713102:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879110.713105:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879110.713108:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879111.417111:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879111.417144:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879111.417164:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879111.417166:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879111.417196:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879111.417199:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879111.417203:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879111.737055:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879111.737061:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879111.737063:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879111.737067:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879111.737085:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879111.737088:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879111.737090:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879111.737092:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879111.737093:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879112.441068:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879112.441079:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879112.441088:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879112.441091:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879112.441167:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879112.441173:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879112.441195:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879112.761073:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879112.761096:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879112.761099:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879112.761108:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879112.761142:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879112.761148:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879112.761153:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879112.761158:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879112.761163:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879113.465090:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879113.465103:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879113.465170:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879113.465176:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879113.465180:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879113.465187:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879113.465203:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879113.465208:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879113.465212:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879113.465216:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879113.465361:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879113.465367:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879113.465371:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879113.721069:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879113.721086:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879113.721090:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879113.721096:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879113.721126:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879113.721137:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10052 00000100:00080000:3.0:1788879113.721158:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879113.721164:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879113.721168:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879113.721185:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879113.721191:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879113.721197:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879113.721203:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879113.721208:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879113.721211:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879113.721220:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879113.721227:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879113.721235:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879113.721243:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879113.721249:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.721254:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879113.721259:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879113.721281:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879113.721284:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.721289:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879113.721294:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879113.721302:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.721321:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879113.721336:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290130816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879113.721352:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879113.721359:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.721387:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879113.721392:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879113.721397:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879113.721399:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879113.721402:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879113.721406:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879113.721410:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10052 00000100:00080000:3.0:1788879113.721415:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879113.721418:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879113.721420:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879113.721425:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879113.721427:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879113.721430:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879113.721434:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879113.721437:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879113.721440:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879113.721444:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879113.721449:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879113.721451:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879113.721460:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879113.721464:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.721468:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879113.721476:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879113.721481:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.721486:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879113.721492:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879113.721494:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.721496:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879113.721498:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879113.721501:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.721509:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.721512:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1788879113.721520:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290130944/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879113.721524:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879113.721531:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879113.721541:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879113.721548:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879113.721550:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.721558:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.721564:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879113.721568:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879113.721577:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879113.721579:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1788879113.721582:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1788879113.721582:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879113.721591:0:136860:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:1.0:1788879113.721592:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879113.721603:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10052 02000000:00000001:1.0:1788879113.721605:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879113.721619:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879113.721631:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1788879113.721632:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290130816:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879113.721638:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879113.721653:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1788879113.721653:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879113.721666:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1788879113.721670:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1788879113.721674:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 02000000:00000001:1.0:1788879113.721676:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879113.721686:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1788879113.721692:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 02000000:00000001:1.0:1788879113.721705:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.721705:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879113.721713:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879113.721729:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879113.721736:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879113.721740:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879113.721742:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788879113.721749:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879113.721750:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 02000000:00000001:3.0:1788879113.721762:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.721767:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879113.721773:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879113.721778:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879113.721782:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.721785:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879113.721792:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879113.721795:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.721798:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879113.721801:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879113.721804:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.721813:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879113.721821:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290131072/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879113.721833:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879113.721838:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879113.721844:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879113.721847:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879113.721854:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879113.721860:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879113.721862:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879113.721866:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879113.721867:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879113.721880:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879113.721883:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1788879113.721888:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10052 00000100:00000001:1.0:1788879113.721890:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.721896:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:3.0:1788879113.721899:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879113.721954:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879113.721957:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879113.721965:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879113.721968:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879113.721970:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879113.721973:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879113.721976:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879113.721979:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879113.721981:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879113.721990:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879113.721994:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304725546 00000100:00000001:3.0:1788879113.721995:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879113.722015:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879113.722022:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879113.722026:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788879113.722029:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.722035:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879113.722039:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879113.722042:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.722045:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879113.722052:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879113.722054:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304725546 02000000:00000001:3.0:1788879113.722055:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.722070:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879113.722077:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1788879113.722077:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879113.722091:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.722095:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879113.722101:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879113.722102:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.722148:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879113.722154:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290131200/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879113.722168:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879113/real 1788879113] req@ffff94e50449fb80 x1875765290130816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879113.722211:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:1.0:1788879113.722216:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788879113.722222:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879113.722223:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879113.722234:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879113.722237:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.722240:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879113.722246:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722251:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879113.722254:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879113.722264:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290130816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879113.722275:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722280:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879113.722285:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722288:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879113.722291:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722296:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879113.722300:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879113.722305:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879113.722310:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879113.722313:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879113.722318:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290130816:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879113.722324:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722327:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879113.722330:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879113.722333:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879113.722338:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879113.722341:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879113.722344:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879113.722349:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879113.722352:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879113.722356:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879113.722358:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722361:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722369:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879113.722371:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.722375:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879113.722377:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879113.722379:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879113.722382:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879113.722384:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879113.722388:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879113.722391:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879113.722394:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879113.722398:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290130944:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879113.722402:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879113.722405:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879113.722407:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879113.722409:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879113.722414:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.722422:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879113.722424:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:1.0:1788879113.722430:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879113.722451:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879113.722457:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879113.722459:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879113.722461:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.722464:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879113.722467:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.722473:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304725004 00000100:00000001:1.0:1788879113.722476:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304725546 00000100:00000001:1.0:1788879113.722480:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304725004 00000100:00000001:1.0:1788879113.722483:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304725546 00000100:00000001:1.0:1788879113.722485:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879113.722488:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.722490:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879113.722495:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879113/real 1788879113] req@ffff94e50449d880 x1875765290130944/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879113.722505:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879113.722507:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879113.722510:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879113.722512:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879113.722513:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722516:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879113.722518:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879113.722522:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290130944/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879113.722530:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722533:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879113.722535:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722537:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879113.722539:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722541:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879113.722558:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879113.722562:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879113.722566:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879113.722569:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879113.722573:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290130944:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879113.722577:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879113.722579:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879113.722581:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879113.722584:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879113.722586:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879113.722588:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879113.722592:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290131072:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879113.722596:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879113.722598:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879113.722600:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879113.722603:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879113.722606:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.722611:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879113.722613:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:1.0:1788879113.722617:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879113.722628:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879113.722633:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879113.722635:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879113.722637:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.722639:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879113.722642:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.722645:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304725002 00000100:00000001:1.0:1788879113.722649:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304725546 00000100:00000001:1.0:1788879113.722651:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304725002 00000100:00000001:1.0:1788879113.722654:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304725546 00000100:00000001:1.0:1788879113.722657:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879113.722660:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.722662:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879113.722667:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879113/real 1788879113] req@ffff94e50449c380 x1875765290131072/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879113.722676:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879113.722678:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879113.722680:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879113.722682:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879113.722683:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722685:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879113.722687:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879113.722691:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290131072/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879113.722702:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722706:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879113.722711:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722715:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879113.722717:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722720:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879113.722742:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879113.722747:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879113.722755:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879113.722760:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879113.722766:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290131072:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879113.722774:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879113.722777:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879113.722780:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879113.722785:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879113.722788:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879113.722792:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879113.722799:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290131200:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879113.722806:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879113.722810:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879113.722813:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879113.722815:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879113.722820:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.722828:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879113.722829:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000100:00000001:1.0:1788879113.722834:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879113.722851:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879113.722857:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879113.722861:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879113.722863:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.722865:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879113.722868:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.722873:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304725002 00000100:00000001:1.0:1788879113.722876:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304725546 00000100:00000001:1.0:1788879113.722879:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304725002 00000100:00000001:1.0:1788879113.722881:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304725546 00000100:00000001:1.0:1788879113.722884:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879113.722886:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.722889:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879113.722894:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879113/real 1788879113] req@ffff94e50449e680 x1875765290131200/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879113.722939:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879113.722940:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879113.722944:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879113.722946:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879113.722948:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722950:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879113.722952:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879113.722958:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290131200/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879113.722975:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722978:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879113.722981:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722983:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879113.722985:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.722987:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879113.723002:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879113.723005:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879113.723008:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879113.723011:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879113.723014:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290131200:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879113.723019:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.723022:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879113.723023:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879113.723026:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879113.723029:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879113.723030:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879113.723032:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879113.723035:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879113.723037:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879113.723040:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879113.723042:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.723044:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879113.723045:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879113.723047:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879113.723049:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879113.723050:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879113.723052:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879113.723054:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879113.723055:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879113.723057:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879113.723058:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.723060:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879113.723062:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879113.723064:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879113.723065:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879113.723067:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879113.723068:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879113.723071:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879113.723072:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879113.723074:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879113.723076:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.723078:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.723082:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879113.723084:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.723086:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879113.723088:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879113.785063:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879113.785072:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879113.785076:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879113.785084:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879113.785091:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879113.785096:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.785101:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879113.785133:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879113.785138:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879114.489054:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879114.489063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879114.489070:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879114.489071:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879114.489126:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879114.489130:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879114.489134:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879114.809037:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879114.809046:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879114.809049:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879114.809056:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879114.809062:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879114.809068:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879114.809071:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879114.809075:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879114.809079:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879115.512984:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879115.512992:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879115.512995:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879115.512998:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879115.513003:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879115.513005:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879115.513008:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879115.513010:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879115.513015:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879115.513016:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879115.513035:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879115.513037:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879115.513039:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879115.832996:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879115.833002:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879115.833004:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879115.833008:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879115.833011:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879115.833014:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879115.833016:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879115.833019:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879115.833021:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879116.537059:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879116.537067:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879116.537073:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879116.537074:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879116.537177:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879116.537180:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879116.537182:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879116.856979:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879116.856985:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879116.856987:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879116.856991:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879116.856993:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879116.856997:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879116.856999:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879116.857001:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879116.857003:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879117.561129:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879117.561139:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879117.561148:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879117.561150:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879117.561216:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879117.561221:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879117.561225:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879117.881054:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879117.881065:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879117.881069:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879117.881101:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879117.881108:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879117.881114:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879117.881119:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879117.881125:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879117.881130:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879118.584981:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879118.585014:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879118.585017:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879118.585019:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879118.585043:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879118.585045:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879118.585047:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879118.841056:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879118.841066:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879118.841088:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879118.841092:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879118.841097:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879118.841104:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10057 00000100:00080000:1.0:1788879118.841116:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879118.841119:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879118.841121:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879118.841129:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879118.841132:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879118.841136:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879118.841138:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879118.841141:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879118.841142:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879118.841147:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879118.841150:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879118.841154:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879118.841158:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879118.841161:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879118.841163:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879118.841166:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879118.841177:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879118.841179:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879118.841182:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879118.841185:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879118.841189:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879118.841201:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879118.841211:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100ad80 x1875765290131328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879118.841223:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879118.841228:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879118.841248:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879118.841252:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879118.841256:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879118.841257:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879118.841259:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879118.841262:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879118.841265:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10057 00000100:00080000:1.0:1788879118.841269:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879118.841271:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879118.841273:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879118.841277:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879118.841279:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879118.841281:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879118.841285:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1788879118.841286:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879118.841292:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879118.841294:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879118.841295:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879118.841299:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1788879118.841303:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879118.841304:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879118.841309:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.841311:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879118.841314:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879118.841320:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879118.841322:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879118.841325:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879118.841327:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879118.841333:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788879118.841333:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1788879118.841340:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879118.841345:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290131328:192.168.203.129@tcp:250:kworker.0 00000008:00000001:1.0:1788879118.841346:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879118.841351:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.841358:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879118.841360:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879118.841364:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879118.841364:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1788879118.841366:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879118.841369:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879118.841370:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879118.841375:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879118.841377:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788879118.841383:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879118.841385:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879118.841393:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008000 x1875765290131456/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879118.841402:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879118.841404:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.841405:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879118.841409:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.841410:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286519680 : -117763423031936 : ffff94e51100ad80) 00000100:00080000:1.0:1788879118.841415:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879118.841417:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1788879118.841421:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879118.841425:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879118.841427:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879118.841430:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879118.841433:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10057 00000100:00080000:1.0:1788879118.841437:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879118.841439:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879118.841440:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879118.841444:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879118.841447:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879118.841450:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879118.841452:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879118.841455:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879118.841456:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879118.841464:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:3.0:1788879118.841466:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879118.841468:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879118.841473:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879118.841476:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879118.841479:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879118.841481:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879118.841483:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00100000:1.0:1788879118.841484:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879118.841489:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.841492:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879118.841493:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.841499:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879118.841501:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879118.841503:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:1.0:1788879118.841505:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879118.841509:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879118.841511:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879118.841512:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879118.841513:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.841518:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304730665 00000100:00000001:1.0:1788879118.841521:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879118.841528:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008e00 x1875765290131584/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879118.841534:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788879118.841536:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879118.841541:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879118.841545:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879118.841547:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879118.841550:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879118.841551:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879118.841553:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879118.841555:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879118.841558:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304730665 00000100:00080000:1.0:1788879118.841559:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10057 00000100:00080000:1.0:1788879118.841567:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879118.841570:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879118.841571:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879118.841577:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:1.0:1788879118.841578:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879118.841581:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879118.841584:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879118.841586:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1788879118.841588:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879118.841594:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879118.841595:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879118.841599:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879118.841600:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1788879118.841607:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879118.841614:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879118.841614:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879118/real 1788879118] req@ffff94e51100ad80 x1875765290131328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788879118.841618:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879118.841623:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879118.841629:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879118.841635:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879118.841639:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000400:00000001:3.0:1788879118.841642:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:1.0:1788879118.841643:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879118.841649:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879118.841653:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879118.841654:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879118.841655:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879118.841656:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879118.841661:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879118.841663:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.841666:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879118.841668:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.841674:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00100000:1.0:1788879118.841678:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008700 x1875765290131712/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879118.841680:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.841690:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879118.841695:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879118.841697:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879118.841699:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879118.841706:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879118.841716:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100ad80 x1875765290131328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879118.841733:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.841740:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879118.841745:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.841748:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879118.841751:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.841753:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879118.841755:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879118.841759:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879118.841764:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879118.841767:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879118.841771:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290131328:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879118.841776:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.841779:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879118.841781:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879118.841784:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879118.841787:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879118.841789:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879118.841791:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879118.841795:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879118.841797:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879118.841800:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879118.841802:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.841804:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.841811:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879118.841813:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.841815:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879118.841817:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879118.841819:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879118.841820:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879118.841822:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879118.841824:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879118.841826:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879118.841829:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879118.841833:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290131456:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879118.841836:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879118.841838:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879118.841840:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879118.841842:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879118.841845:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.841853:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879118.841854:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508032 : -117763423043584 : ffff94e511008000) 00000100:00000001:3.0:1788879118.841859:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879118.841875:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879118.841880:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879118.841882:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879118.841884:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.841885:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879118.841888:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.841892:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304730123 00000100:00000001:3.0:1788879118.841895:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304730665 00000100:00000001:3.0:1788879118.841897:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304730123 00000100:00000001:3.0:1788879118.841900:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304730665 00000100:00000001:3.0:1788879118.841928:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879118.841931:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.841933:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879118.841938:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879118/real 1788879118] req@ffff94e511008000 x1875765290131456/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879118.841950:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879118.841953:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879118.841957:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879118.841960:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879118.841962:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.841965:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879118.841967:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879118.841973:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008000 x1875765290131456/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879118.841995:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.841999:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879118.842003:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.842006:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879118.842008:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.842012:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879118.842029:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879118.842032:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879118.842036:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879118.842038:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879118.842042:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290131456:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879118.842047:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879118.842048:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879118.842050:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879118.842052:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879118.842054:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879118.842057:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879118.842060:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290131584:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879118.842063:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879118.842065:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879118.842089:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879118.842094:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879118.842098:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.842114:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879118.842115:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286511616 : -117763423040000 : ffff94e511008e00) 00000100:00000001:3.0:1788879118.842119:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879118.842135:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879118.842140:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879118.842143:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879118.842144:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.842147:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879118.842150:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.842153:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304730122 00000100:00000001:3.0:1788879118.842156:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304730665 00000100:00000001:3.0:1788879118.842158:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304730122 00000100:00000001:3.0:1788879118.842161:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304730665 00000100:00000001:3.0:1788879118.842163:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879118.842165:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.842166:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879118.842171:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879118/real 1788879118] req@ffff94e511008e00 x1875765290131584/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879118.842179:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879118.842180:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879118.842182:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879118.842184:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879118.842185:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.842187:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879118.842188:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879118.842192:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008e00 x1875765290131584/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879118.842199:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.842202:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879118.842204:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.842206:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879118.842207:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.842209:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879118.842219:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879118.842221:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879118.842224:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879118.842226:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879118.842230:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290131584:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879118.842233:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879118.842235:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879118.842236:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879118.842239:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879118.842241:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879118.842243:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879118.842245:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290131712:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879118.842249:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879118.842250:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879118.842252:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879118.842254:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879118.842257:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.842260:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879118.842261:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286509824 : -117763423041792 : ffff94e511008700) 00000100:00000001:3.0:1788879118.842264:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879118.842274:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879118.842278:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879118.842280:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879118.842281:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.842283:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879118.842286:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.842289:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304730122 00000100:00000001:3.0:1788879118.842291:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304730665 00000100:00000001:3.0:1788879118.842293:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304730122 00000100:00000001:3.0:1788879118.842296:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304730665 00000100:00000001:3.0:1788879118.842298:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879118.842299:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.842301:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879118.842305:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879118/real 1788879118] req@ffff94e511008700 x1875765290131712/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879118.842319:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879118.842320:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879118.842322:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879118.842323:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879118.842325:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.842326:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879118.842327:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879118.842331:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008700 x1875765290131712/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879118.842337:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.842339:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879118.842341:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.842343:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879118.842344:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.842346:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879118.842356:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879118.842358:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879118.842361:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879118.842363:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879118.842366:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290131712:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879118.842370:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.842372:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879118.842373:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879118.842375:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879118.842377:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879118.842378:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879118.842379:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879118.842382:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879118.842383:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879118.842385:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879118.842387:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.842389:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879118.842390:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879118.842391:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879118.842393:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879118.842394:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879118.842396:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879118.842397:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879118.842398:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879118.842400:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879118.842401:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.842403:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879118.842404:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879118.842405:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879118.842406:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879118.842407:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879118.842408:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879118.842410:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879118.842411:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879118.842412:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879118.842413:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.842415:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879118.842418:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879118.842420:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879118.842422:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879118.842423:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879118.904991:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879118.904999:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879118.905001:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879118.905007:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879118.905011:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879118.905015:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879118.905017:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879118.905020:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879118.905022:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879119.609092:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879119.609101:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879119.609128:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879119.609131:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879119.609167:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879119.609173:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879119.609178:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879119.929161:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879119.929169:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879119.929171:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879119.929177:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879119.929182:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879119.929186:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879119.929189:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879119.929192:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879119.929195:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879120.633091:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879120.633099:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879120.633106:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879120.633108:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879120.633136:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879120.633140:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879120.633143:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879120.952978:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879120.952985:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879120.952987:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879120.952992:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879120.952996:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879120.952999:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879120.953002:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879120.953004:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879120.953008:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879121.656995:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879121.657001:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879121.657005:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879121.657006:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879121.657091:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879121.657096:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879121.657102:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879121.977062:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879121.977077:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879121.977079:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879121.977085:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879121.977089:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879121.977093:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879121.977096:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879121.977099:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879121.977112:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879122.681062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879122.681073:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879122.681083:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879122.681085:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879122.681117:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879122.681120:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879122.681123:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879123.001023:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879123.001031:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879123.001033:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879123.001055:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879123.001060:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879123.001063:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.001066:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879123.001070:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.001072:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879123.705111:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879123.705118:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879123.705123:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879123.705126:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879123.705128:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879123.705131:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879123.705134:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879123.705135:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879123.705136:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879123.705138:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879123.705176:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879123.705179:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879123.705181:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879123.961228:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879123.961248:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879123.961255:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879123.961264:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879123.961277:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879123.961294:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10062 00000100:00080000:3.0:1788879123.961342:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879123.961347:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879123.961350:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879123.961363:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879123.961369:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879123.961375:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879123.961379:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879123.961382:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879123.961384:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879123.961391:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879123.961395:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879123.961399:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879123.961405:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879123.961410:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879123.961414:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879123.961419:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879123.961438:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879123.961441:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879123.961447:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879123.961453:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879123.961460:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879123.961480:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879123.961495:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290131840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879123.961512:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879123.961520:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879123.961570:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879123.961575:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879123.961581:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879123.961583:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879123.961586:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879123.961590:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879123.961595:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10062 00000100:00080000:3.0:1788879123.961603:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879123.961608:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879123.961612:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879123.961621:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788879123.961622:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879123.961631:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879123.961632:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879123.961639:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879123.961644:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879123.961647:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879123.961690:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879123.961696:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879123.961700:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879123.961712:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290131840:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879123.961720:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879123.961727:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879123.961731:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879123.961735:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879123.961742:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.961788:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879123.961792:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:1.0:1788879123.961799:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1788879123.961836:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879123.961843:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879123.961847:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879123.961850:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879123.961858:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:1.0:1788879123.961870:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879123.961871:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879123.961881:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879123.961891:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879123.961896:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.961897:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879123.961900:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879123.961909:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879123.961915:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.961962:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879123.961971:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.961979:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304735786 00000100:00000001:1.0:1788879123.961996:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788879123.962007:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304735786 00000100:00000001:1.0:1788879123.962015:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879123.962023:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879123.962027:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.962058:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879123.962093:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879123/real 1788879123] req@ffff94e50449df80 x1875765290131840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879123.962138:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879123.962142:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879123.962156:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879123.962165:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879123.962172:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.962178:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879123.962182:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879123.962192:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290131840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879123.962203:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.962210:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879123.962215:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.962218:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879123.962221:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.962225:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879123.962228:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879123.962233:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879123.962239:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879123.962242:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879123.962249:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290131840:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879123.962260:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.962265:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879123.962268:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879123.962273:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879123.962280:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879123.962284:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879123.962293:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000008:00100000:3.0:1788879123.962296:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879123.962301:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000008:00000001:3.0:1788879123.962304:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879123.962310:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879123.962310:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.962343:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879123.962350:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.962354:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.962367:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879123.962371:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.962377:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879123.962380:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879123.962415:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879123.962420:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879123.962427:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879123.962432:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879123.962437:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879123.962451:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879123.962464:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290131968/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879123.962477:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879123.962482:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879123.962507:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879123.962512:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879123.962517:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879123.962519:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879123.962521:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879123.962526:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879123.962530:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10062 00000100:00080000:3.0:1788879123.962540:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879123.962543:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879123.962547:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879123.962552:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788879123.962553:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879123.962561:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879123.962562:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788879123.962566:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879123.962568:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788879123.962572:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879123.962574:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879123.962578:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879123.962580:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879123.962584:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879123.962591:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1788879123.962593:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788879123.962598:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879123.962600:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879123.962607:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879123.962609:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879123.962621:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788879123.962622:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290131968:192.168.203.129@tcp:38:kworker.0 02000000:00000001:3.0:1788879123.962652:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.962654:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879123.962661:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879123.962663:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879123.962669:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788879123.962672:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879123.962673:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879123.962681:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788879123.962683:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879123.962690:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.962696:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788879123.962701:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879123.962702:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 02000000:00000001:3.0:1788879123.962709:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.962713:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879123.962717:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879123.962724:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879123.962727:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879123.962738:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879123.962745:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1788879123.962771:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290132096/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879123.962778:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879123.962785:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879123.962787:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879123.962790:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879123.962794:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.962797:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879123.962805:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.962807:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879123.962811:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879123.962817:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304735244 00000100:00080000:3.0:1788879123.962818:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879123.962826:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879123.962829:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304735786 00000100:00080000:3.0:1788879123.962831:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879123.962842:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304735244 00000100:00000001:3.0:1788879123.962844:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879123.962854:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10062 00000100:00000001:1.0:1788879123.962855:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304735786 00000100:00080000:3.0:1788879123.962862:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879123.962863:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879123.962871:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879123.962873:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879123.962876:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879123.962881:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1788879123.962884:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879123.962888:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879123.962894:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:1.0:1788879123.962896:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879123/real 1788879123] req@ffff94e50449ce00 x1875765290131968/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788879123.962899:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879123.962909:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879123.962914:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879123.962929:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879123.962980:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879123.962986:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:1.0:1788879123.962987:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879123.962993:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788879123.962994:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879123.962999:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:3.0:1788879123.963002:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.963009:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879123.963009:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879123.963013:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879123.963015:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1788879123.963021:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.963023:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879123.963028:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.963029:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879123.963084:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290131968/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879123.963084:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879123.963090:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879123.963100:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879123.963105:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879123.963111:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.963117:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879123.963127:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.963128:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879123.963140:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879123.963146:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290132224/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879123.963148:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879123.963154:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.963164:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879123.963172:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879123.963176:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879123.963183:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.963199:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879123.963206:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879123.963214:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879123.963218:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879123.963224:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290131968:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879123.963230:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.963234:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879123.963236:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879123.963238:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879123.963242:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879123.963246:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879123.963249:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879123.963254:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879123.963257:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879123.963262:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879123.963265:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.963285:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.963298:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879123.963301:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.963306:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879123.963309:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879123.963312:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879123.963315:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879123.963318:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879123.963340:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879123.963343:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879123.963347:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879123.963355:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290132096:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879123.963360:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879123.963363:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879123.963366:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879123.963368:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879123.963373:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.963383:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879123.963385:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:1.0:1788879123.963391:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879123.963411:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879123.963418:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879123.963421:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879123.963423:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.963426:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879123.963429:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.963434:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304735243 00000100:00000001:1.0:1788879123.963438:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304735786 00000100:00000001:1.0:1788879123.963441:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304735243 00000100:00000001:1.0:1788879123.963444:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304735786 00000100:00000001:1.0:1788879123.963447:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879123.963449:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.963451:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879123.963457:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879123/real 1788879123] req@ffff94e50449df80 x1875765290132096/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879123.963477:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879123.963479:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879123.963482:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879123.963484:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879123.963486:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.963489:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879123.963491:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879123.963496:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290132096/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879123.963504:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.963507:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879123.963510:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.963512:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879123.963514:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.963516:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879123.963535:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879123.963538:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879123.963541:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879123.963544:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879123.963548:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290132096:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879123.963553:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879123.963555:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879123.963557:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879123.963559:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879123.963562:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879123.963564:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879123.963568:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290132224:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879123.963571:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879123.963573:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879123.963576:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879123.963579:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879123.963582:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.963588:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879123.963589:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000100:00000001:1.0:1788879123.963594:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879123.963606:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879123.963611:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879123.963613:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879123.963615:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.963617:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879123.963620:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.963624:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304735243 00000100:00000001:1.0:1788879123.963627:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304735786 00000100:00000001:1.0:1788879123.963631:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304735243 00000100:00000001:1.0:1788879123.963634:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304735786 00000100:00000001:1.0:1788879123.963637:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879123.963639:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.963658:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879123.963662:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879123/real 1788879123] req@ffff94e50449e680 x1875765290132224/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879123.963671:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879123.963672:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879123.963675:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879123.963676:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879123.963678:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.963680:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879123.963682:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879123.963686:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290132224/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879123.963695:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.963697:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879123.963701:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.963703:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879123.963704:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.963707:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879123.963725:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879123.963728:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879123.963732:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879123.963735:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879123.963738:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290132224:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879123.963742:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.963744:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879123.963746:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879123.963748:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879123.963750:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879123.963752:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879123.963753:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879123.963756:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879123.963757:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879123.963759:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879123.963761:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.963763:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879123.963765:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879123.963766:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879123.963769:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879123.963770:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879123.963772:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879123.963773:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879123.963774:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879123.963776:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879123.963777:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.963779:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879123.963783:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879123.963784:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879123.963786:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879123.963788:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879124.025069:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879124.025078:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879124.025082:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879124.025091:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879124.025098:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879124.025105:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879124.025109:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879124.025114:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879124.025119:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879124.729009:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879124.729019:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879124.729040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879124.729042:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879124.729071:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879124.729074:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879124.729077:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879125.048962:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879125.048971:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879125.048973:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879125.048979:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879125.048982:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879125.048994:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879125.048997:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879125.049000:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879125.049003:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879125.753036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879125.753050:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879125.753053:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879125.753057:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879125.753064:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879125.753067:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879125.753069:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879125.753072:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879125.753076:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879125.753077:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879125.753180:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879125.753189:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879125.753195:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879126.073096:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879126.073107:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879126.073110:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879126.073118:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879126.073123:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879126.073130:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879126.073134:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879126.073138:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879126.073142:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879126.777084:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879126.777093:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879126.777101:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879126.777103:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879126.777151:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879126.777155:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879126.777160:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879127.097144:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879127.097155:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879127.097158:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879127.097166:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879127.097172:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879127.097179:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879127.097184:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879127.097190:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879127.097196:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879127.801102:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879127.801111:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879127.801118:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879127.801119:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879127.801173:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879127.801176:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879127.801180:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879128.121079:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879128.121089:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879128.121092:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879128.121099:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879128.121105:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879128.121111:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879128.121115:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879128.121119:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879128.121122:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879128.825056:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879128.825064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879128.825071:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879128.825073:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879128.825108:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879128.825111:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879128.825115:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879129.081202:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879129.081218:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879129.081222:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879129.081228:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879129.081235:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879129.081247:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10067 00000100:00080000:1.0:1788879129.081264:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879129.081285:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879129.081288:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879129.081300:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879129.081304:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879129.081309:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879129.081312:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879129.081315:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879129.081317:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879129.081323:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879129.081325:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879129.081329:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879129.081334:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879129.081338:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879129.081358:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879129.081361:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879129.081383:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879129.081385:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879129.081389:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879129.081393:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879129.081401:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879129.081421:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879129.081435:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100bb80 x1875765290132352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879129.081450:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879129.081456:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879129.081492:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879129.081497:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879129.081502:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879129.081504:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879129.081527:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879129.081531:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879129.081534:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10067 00000100:00080000:1.0:1788879129.081541:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879129.081544:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879129.081545:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879129.081550:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879129.081553:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879129.081557:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879129.081561:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879129.081564:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879129.081566:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879129.081570:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879129.081573:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879129.081576:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879129.081578:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879129.081600:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1788879129.081603:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879129.081605:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1788879129.081612:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.081615:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879129.081619:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879129.081622:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1788879129.081632:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788879129.081634:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:1.0:1788879129.081643:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879129.081643:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879129.081662:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879129.081671:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879129.081680:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1788879129.081682:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290132352:192.168.203.129@tcp:250:kworker.0 02000000:00000001:1.0:1788879129.081685:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879129.081695:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879129.081704:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879129.081707:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879129.081710:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879129.081715:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879129.081721:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879129.081722:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879129.081736:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1788879129.081745:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100aa00 x1875765290132480/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879129.081750:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879129.081772:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879129.081776:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879129.081783:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.081786:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1788879129.081788:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879129.081793:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286523264 : -117763423028352 : ffff94e51100bb80) 00000100:00080000:1.0:1788879129.081796:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879129.081802:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788879129.081803:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1788879129.081813:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879129.081819:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879129.081824:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10067 00000100:00080000:1.0:1788879129.081833:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879129.081836:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879129.081838:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879129.081859:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879129.081862:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879129.081866:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000800:00000100:3.0:1788879129.081872:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:1.0:1788879129.081877:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879129.081884:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879129.081887:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879129.081892:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:1.0:1788879129.081895:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879129.081934:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879129.081939:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.081942:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879129.081950:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.081956:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304740906 00000100:00000001:1.0:1788879129.081983:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.081984:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1788879129.081993:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879129.082020:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879129.082033:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879129.082038:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879129.082061:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879129.082061:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304740906 00000100:00000001:3.0:1788879129.082072:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000008:00000001:1.0:1788879129.082074:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.082080:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879129.082085:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.082087:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.082103:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1788879129.082111:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879129.082115:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879129.082115:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879129/real 1788879129] req@ffff94e51100bb80 x1875765290132352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879129.082125:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879129.082134:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879129.082142:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879129.082161:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879129.082162:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879129.082176:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879129.082182:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00100000:1.0:1788879129.082184:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b480 x1875765290132608/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879129.082189:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879129.082194:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.082209:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879129.082217:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879129.082221:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879129.082229:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879129.082232:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100bb80 x1875765290132352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879129.082237:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879129.082253:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879129.082278:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788879129.082291:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879129.082293:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879129.082301:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788879129.082308:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788879129.082309:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879129.082329:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879129.082331:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879129.082338:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788879129.082348:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10067 00000100:00000001:3.0:1788879129.082350:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879129.082365:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879129.082381:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879129.082383:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879129.082399:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879129.082406:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879129.082407:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879129.082426:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:1.0:1788879129.082427:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879129.082432:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879129.082439:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:3.0:1788879129.082442:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290132352:192.168.203.129@tcp:250:kworker.0 00000020:00000001:1.0:1788879129.082450:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879129.082456:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879129.082458:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879129.082462:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879129.082463:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879129.082473:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00080000:1.0:1788879129.082480:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:3.0:1788879129.082481:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1788879129.082492:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879129.082494:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879129.082503:0:2361:0:(genops.c:1216:class_import_put()) Process entered 02000000:00000001:1.0:1788879129.082522:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:3.0:1788879129.082522:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879129.082541:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879129.082542:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879129.082553:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:1.0:1788879129.082555:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.082565:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879129.082566:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879129.082587:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:1.0:1788879129.082593:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879129.082595:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1788879129.082606:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879129.082611:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.082616:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:1.0:1788879129.082619:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879129.082621:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879129.082624:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879129.082630:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879129.082632:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1788879129.082647:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879129.082649:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879129.082655:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1788879129.082657:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.082660:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879129.082664:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879129.082668:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879129.082670:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879129.082673:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879129.082682:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290132480:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879129.082684:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.082697:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879129.082703:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879129.082717:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879129.082724:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1788879129.082727:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b800 x1875765290132736/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879129.082731:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.082751:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879129.082754:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:1.0:1788879129.082760:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879129.082768:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879129.082769:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879129.082788:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788879129.082808:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879129.082816:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879129.082822:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879129.082824:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.082828:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879129.082834:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.082840:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304740364 00000100:00000001:3.0:1788879129.082844:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304740906 00000100:00000001:3.0:1788879129.082847:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304740364 00000100:00000001:3.0:1788879129.082850:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304740906 00000100:00000001:3.0:1788879129.082853:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879129.082856:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.082858:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879129.082864:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879129/real 1788879129] req@ffff94e51100aa00 x1875765290132480/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879129.082885:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879129.082887:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879129.082890:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879129.082892:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879129.082894:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.082896:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879129.082898:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879129.082934:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290132480/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879129.082943:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.082947:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879129.082953:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.082956:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879129.082958:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.082962:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879129.082994:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879129.083019:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879129.083027:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879129.083032:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879129.083039:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290132480:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879129.083048:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879129.083051:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879129.083066:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879129.083071:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879129.083074:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879129.083078:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879129.083084:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290132608:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879129.083091:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879129.083095:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879129.083099:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879129.083102:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879129.083109:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.083121:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879129.083123:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:3.0:1788879129.083130:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879129.083157:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879129.083167:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879129.083171:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879129.083174:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.083176:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879129.083181:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.083186:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304740363 00000100:00000001:3.0:1788879129.083190:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304740906 00000100:00000001:3.0:1788879129.083193:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304740363 00000100:00000001:3.0:1788879129.083196:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304740906 00000100:00000001:3.0:1788879129.083199:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879129.083201:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.083204:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879129.083210:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879129/real 1788879129] req@ffff94e51100b480 x1875765290132608/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879129.083219:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879129.083221:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879129.083224:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879129.083226:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879129.083228:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.083230:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879129.083232:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879129.083236:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290132608/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879129.083245:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.083247:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879129.083251:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.083253:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879129.083255:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.083257:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879129.083271:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879129.083275:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879129.083279:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879129.083282:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879129.083286:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290132608:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879129.083297:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.083299:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879129.083301:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879129.083303:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879129.083305:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879129.083307:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879129.083309:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879129.083312:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879129.083314:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879129.083316:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879129.083318:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.083320:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879129.083322:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879129.083323:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879129.083325:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879129.083327:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879129.083328:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879129.083330:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879129.083331:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879129.083333:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879129.083334:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.083337:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.083341:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879129.083343:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.083346:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879129.083348:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879129.083360:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879129.083361:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879129.083364:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879129.083367:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879129.083369:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879129.083372:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879129.083375:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290132736:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879129.083379:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879129.083382:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879129.083384:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879129.083386:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879129.083390:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.083396:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879129.083398:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:3.0:1788879129.083402:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879129.083417:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879129.083422:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879129.083425:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879129.083427:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.083429:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879129.083432:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.083436:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304740363 00000100:00000001:3.0:1788879129.083439:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304740906 00000100:00000001:3.0:1788879129.083442:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304740363 00000100:00000001:3.0:1788879129.083445:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304740906 00000100:00000001:3.0:1788879129.083448:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879129.083450:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.083452:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879129.083457:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879129/real 1788879129] req@ffff94e51100b800 x1875765290132736/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879129.083466:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879129.083467:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879129.083470:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879129.083472:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879129.083474:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.083477:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879129.083479:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879129.083484:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290132736/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879129.083492:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.083495:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879129.083497:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.083499:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879129.083501:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.083503:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879129.083526:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879129.083529:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879129.083532:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879129.083535:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879129.083538:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290132736:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879129.083542:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.083544:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879129.083546:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879129.083548:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879129.083550:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879129.083551:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879129.083553:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879129.083556:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879129.083557:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879129.083559:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879129.083561:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.083563:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879129.083566:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879129.083568:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879129.083571:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879129.083572:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879129.145116:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879129.145126:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879129.145129:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879129.145135:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879129.145140:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879129.145145:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879129.145148:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879129.145151:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879129.145154:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879129.849083:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879129.849090:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879129.849095:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879129.849096:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879129.849159:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879129.849165:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879129.849169:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879130.169070:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879130.169080:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879130.169082:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879130.169088:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879130.169091:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879130.169096:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879130.169099:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879130.169102:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879130.169104:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879130.873013:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879130.873018:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879130.873022:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879130.873023:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879130.873068:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879130.873070:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879130.873074:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879131.193057:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879131.193068:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879131.193071:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879131.193078:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879131.193083:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879131.193089:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879131.193092:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879131.193096:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879131.193099:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879131.897035:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879131.897044:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879131.897050:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879131.897052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879131.897112:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879131.897116:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879131.897120:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879132.217011:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879132.217021:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879132.217041:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879132.217049:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879132.217054:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879132.217059:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879132.217063:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879132.217067:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879132.217070:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879132.921080:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879132.921090:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879132.921099:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879132.921102:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879132.921142:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879132.921145:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879132.921148:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879133.241056:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879133.241068:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879133.241073:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879133.241083:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879133.241090:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879133.241095:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879133.241098:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879133.241102:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879133.241105:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879133.945039:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879133.945044:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879133.945048:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879133.945051:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879133.945053:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879133.945056:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879133.945058:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879133.945060:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879133.945062:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879133.945063:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879133.945125:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879133.945129:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879133.945132:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879134.201013:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879134.201021:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879134.201023:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879134.201027:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879134.201030:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879134.201036:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10073 00000100:00080000:2.0:1788879134.201045:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879134.201047:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879134.201049:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879134.201055:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879134.201058:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879134.201060:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879134.201062:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879134.201064:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879134.201065:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879134.201068:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879134.201070:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879134.201072:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879134.201075:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879134.201077:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879134.201079:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879134.201080:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879134.201090:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879134.201092:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879134.201095:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879134.201097:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879134.201100:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879134.201110:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879134.201118:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290132864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879134.201125:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879134.201129:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879134.201141:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879134.201144:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879134.201146:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879134.201147:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879134.201149:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879134.201151:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879134.201153:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10073 00000100:00080000:2.0:1788879134.201155:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879134.201157:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879134.201158:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879134.201161:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879134.201162:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879134.201164:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879134.201166:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879134.201167:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879134.201168:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879134.201171:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879134.201173:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879134.201174:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879134.201177:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879134.201179:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879134.201182:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.201183:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879134.201187:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879134.201188:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879134.201192:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:2.0:1788879134.201193:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879134.201195:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788879134.201200:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879134.201201:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879134.201203:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879134.201205:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879134.201208:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879134.201209:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879134.201211:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879134.201215:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1788879134.201216:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290132864:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879134.201217:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879134.201221:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.201224:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788879134.201226:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879134.201228:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879134.201232:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879134.201233:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290132992/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879134.201235:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879134.201242:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788879134.201243:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879134.201245:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879134.201250:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879134.201252:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879134.201255:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879134.201256:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879134.201257:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879134.201259:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879134.201261:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10073 00000100:00080000:2.0:1788879134.201263:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879134.201264:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879134.201265:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879134.201268:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879134.201270:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 00000020:00000001:2.0:1788879134.201271:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879134.201272:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879134.201275:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879134.201292:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879134.201293:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879134.201294:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879134.201298:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879134.201301:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879134.201303:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879134.201304:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879134.201307:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879134.201308:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879134.201310:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879134.201312:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879134.201314:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879134.201316:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879134.201319:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879134.201321:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879134.201322:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879134.201323:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879134.201325:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879134.201330:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879134.201333:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290133120/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788879134.201340:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879134.201345:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879134.201348:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879134.201351:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879134.201354:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879134.201356:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788879134.201357:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879134.201361:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879134.201362:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879134.201363:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879134.201365:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879134.201367:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879134.201371:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10073 00000100:00000001:3.0:1788879134.201371:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:2.0:1788879134.201377:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879134.201379:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879134.201380:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879134.201384:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879134.201387:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304746025 00000020:00000001:2.0:1788879134.201388:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879134.201391:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879134.201394:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879134.201396:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879134.201397:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879134.201400:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879134.201400:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1788879134.201407:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879134.201409:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.201411:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304746025 02000000:00000001:2.0:1788879134.201412:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879134.201416:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879134.201419:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.201419:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879134.201425:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879134.201426:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00100000:2.0:1788879134.201427:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879134.201432:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788879134.201433:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.201438:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879134.201439:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879134.201445:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879134.201447:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879134.201449:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1788879134.201449:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879134/real 1788879134] req@ffff94e5045e5180 x1875765290132864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879134.201452:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879134.201455:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879134.201461:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879134.201470:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290133248/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788879134.201482:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788879134.201483:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:3.0:1788879134.201485:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879134.201486:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879134.201491:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.201491:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879134.201496:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879134.201498:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.201501:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879134.201502:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879134.201508:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290132864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879134.201517:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.201521:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879134.201525:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.201527:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879134.201528:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.201531:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879134.201533:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879134.201538:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879134.201543:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879134.201546:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879134.201551:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290132864:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879134.201557:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.201560:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879134.201562:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879134.201564:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879134.201584:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879134.201587:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879134.201589:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879134.201594:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879134.201595:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879134.201609:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879134.201611:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.201613:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.201621:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879134.201623:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.201626:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879134.201628:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879134.201630:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879134.201632:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879134.201633:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879134.201637:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879134.201639:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879134.201642:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879134.201647:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290132992:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879134.201652:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879134.201655:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879134.201657:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879134.201659:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879134.201663:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.201672:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879134.201674:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:3.0:1788879134.201679:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879134.201694:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879134.201699:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879134.201701:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879134.201702:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.201704:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879134.201706:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.201710:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304745483 00000100:00000001:3.0:1788879134.201713:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304746025 00000100:00000001:3.0:1788879134.201715:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304745483 00000100:00000001:3.0:1788879134.201717:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304746025 00000100:00000001:3.0:1788879134.201719:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879134.201720:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.201722:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879134.201725:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879134/real 1788879134] req@ffff94e5045e5880 x1875765290132992/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879134.201738:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879134.201739:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879134.201742:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879134.201743:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879134.201744:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.201746:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879134.201747:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879134.201750:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290132992/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879134.201756:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.201758:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879134.201760:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.201761:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879134.201762:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.201763:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879134.201774:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879134.201776:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879134.201779:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879134.201780:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879134.201783:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290132992:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879134.201785:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879134.201786:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879134.201788:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879134.201790:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879134.201791:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879134.201793:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879134.201795:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290133120:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879134.201798:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879134.201799:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879134.201801:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879134.201803:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879134.201805:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.201809:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879134.201810:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:3.0:1788879134.201813:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879134.201821:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879134.201824:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879134.201825:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879134.201827:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.201838:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879134.201840:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.201843:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304745481 00000100:00000001:3.0:1788879134.201845:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304746025 00000100:00000001:3.0:1788879134.201847:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304745481 00000100:00000001:3.0:1788879134.201849:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304746025 00000100:00000001:3.0:1788879134.201851:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879134.201853:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.201854:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879134.201857:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879134/real 1788879134] req@ffff94e5045e6a00 x1875765290133120/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879134.201863:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879134.201864:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879134.201866:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879134.201867:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879134.201868:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.201870:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879134.201871:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879134.201891:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290133120/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879134.201895:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.201896:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879134.201899:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.201900:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879134.201901:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.201902:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879134.201909:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879134.201911:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879134.201914:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879134.201915:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879134.201917:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290133120:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879134.201941:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879134.201943:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879134.201944:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879134.201945:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879134.201947:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879134.201948:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879134.201950:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290133248:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879134.201953:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879134.201954:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879134.201955:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879134.201957:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879134.201959:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.201978:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879134.201979:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:3.0:1788879134.201982:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879134.201992:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879134.201995:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879134.201997:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879134.201998:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.202000:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879134.202002:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.202004:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304745482 00000100:00000001:3.0:1788879134.202006:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304746025 00000100:00000001:3.0:1788879134.202008:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304745482 00000100:00000001:3.0:1788879134.202010:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304746025 00000100:00000001:3.0:1788879134.202012:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879134.202013:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.202014:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879134.202017:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879134/real 1788879134] req@ffff94e5045e6680 x1875765290133248/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879134.202022:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879134.202023:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879134.202024:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879134.202025:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879134.202026:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.202028:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879134.202029:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879134.202031:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290133248/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879134.202036:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.202037:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879134.202039:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.202040:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879134.202041:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.202042:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879134.202051:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879134.202052:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879134.202054:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879134.202056:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879134.202058:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290133248:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879134.202060:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.202062:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879134.202063:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879134.202064:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879134.202066:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879134.202067:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879134.202068:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879134.202070:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879134.202070:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879134.202072:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879134.202073:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.202074:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879134.202075:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879134.202076:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879134.202077:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879134.202078:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879134.202079:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879134.202080:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879134.202080:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879134.202081:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879134.202082:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.202083:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879134.202084:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879134.202085:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879134.202086:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879134.202086:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879134.202087:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879134.202088:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879134.202089:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879134.202089:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879134.202090:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.202091:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879134.202094:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879134.202095:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879134.202096:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879134.202097:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879134.264954:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879134.264959:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879134.264974:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879134.264979:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879134.264982:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879134.264985:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879134.264988:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879134.264990:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879134.264992:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879134.969052:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879134.969059:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879134.969064:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879134.969065:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879134.969082:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879134.969084:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879134.969086:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879135.289064:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879135.289071:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879135.289091:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879135.289096:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879135.289101:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879135.289104:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879135.289107:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879135.289110:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879135.289113:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879135.993103:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879135.993131:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879135.993135:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879135.993141:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879135.993150:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879135.993154:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879135.993158:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879135.993163:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879135.993170:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879135.993173:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879135.993213:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879135.993216:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879135.993220:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879136.313057:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879136.313066:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879136.313068:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879136.313074:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879136.313079:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879136.313083:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879136.313087:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879136.313089:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879136.313093:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879137.017029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879137.017035:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879137.017041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879137.017042:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879137.017062:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879137.017064:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879137.017067:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879137.337026:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879137.337034:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879137.337036:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879137.337042:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879137.337047:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879137.337050:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879137.337053:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879137.337056:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879137.337059:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879138.041036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879138.041042:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879138.041046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879138.041047:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879138.041083:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879138.041104:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879138.041108:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879138.361037:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879138.361047:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879138.361050:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879138.361058:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879138.361063:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879138.361068:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879138.361072:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879138.361076:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879138.361081:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879139.065076:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879139.065087:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879139.065096:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879139.065099:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879139.065133:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879139.065138:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879139.065142:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879139.321002:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879139.321012:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879139.321014:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879139.321018:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879139.321022:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879139.321029:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10078 00000100:00080000:2.0:1788879139.321040:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879139.321043:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879139.321045:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879139.321053:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879139.321055:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879139.321060:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879139.321062:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879139.321064:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879139.321065:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879139.321069:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879139.321071:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879139.321074:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879139.321078:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879139.321080:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879139.321083:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879139.321085:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879139.321096:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879139.321097:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879139.321100:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879139.321103:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879139.321108:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879139.321119:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879139.321128:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290133376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879139.321138:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879139.321142:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879139.321159:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879139.321162:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879139.321165:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879139.321167:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879139.321168:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879139.321171:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879139.321173:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10078 00000100:00080000:2.0:1788879139.321176:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879139.321178:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879139.321179:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879139.321182:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879139.321184:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879139.321186:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879139.321188:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879139.321191:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879139.321193:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879139.321196:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879139.321198:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879139.321200:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879139.321205:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879139.321207:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879139.321208:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879139.321213:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879139.321214:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1788879139.321218:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879139.321219:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879139.321224:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:2.0:1788879139.321225:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321228:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879139.321230:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879139.321238:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879139.321239:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879139.321244:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879139.321244:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879139.321251:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879139.321253:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879139.321255:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879139.321260:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879139.321262:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290133376:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879139.321266:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321269:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879139.321274:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:2.0:1788879139.321276:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290133504/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879139.321278:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879139.321284:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879139.321292:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788879139.321293:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879139.321297:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879139.321302:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879139.321305:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879139.321308:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879139.321309:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879139.321311:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879139.321313:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879139.321314:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879139.321318:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00080000:2.0:1788879139.321321:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10078 00000100:00000001:3.0:1788879139.321327:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788879139.321328:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879139.321333:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879139.321334:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879139.321337:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879139.321338:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879139.321340:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879139.321342:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879139.321344:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879139.321345:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879139.321348:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879139.321351:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879139.321354:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879139.321358:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879139.321362:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879139.321365:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879139.321368:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879139.321371:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879139.321373:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788879139.321379:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:2.0:1788879139.321380:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879139.321385:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879139.321389:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879139.321392:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879139.321393:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:2.0:1788879139.321396:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321399:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879139.321403:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321405:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879139.321407:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321411:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321417:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304751145 00000100:00100000:2.0:1788879139.321421:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290133632/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879139.321429:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788879139.321439:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879139.321442:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879139.321445:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321447:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304751145 00000100:00080000:2.0:1788879139.321448:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879139.321458:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879139.321458:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879139.321465:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788879139.321467:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788879139.321469:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879139.321471:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321478:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879139.321479:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879139.321485:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10078 00000100:00100000:3.0:1788879139.321490:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879139/real 1788879139] req@ffff94e5045e4700 x1875765290133376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879139.321494:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879139.321501:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879139.321505:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:3.0:1788879139.321510:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1788879139.321511:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000001:3.0:1788879139.321515:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1788879139.321516:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879139.321521:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879139.321522:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000020:00000001:2.0:1788879139.321524:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879139.321528:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879139.321528:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879139.321532:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879139.321533:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879139.321539:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879139.321543:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879139.321544:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879139.321550:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879139.321553:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1788879139.321553:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290133376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879139.321560:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879139.321567:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879139.321575:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879139.321575:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1788879139.321578:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879139.321581:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:2.0:1788879139.321585:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321587:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879139.321591:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321592:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879139.321595:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.321599:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1788879139.321601:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879139.321602:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788879139.321606:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879139.321608:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788879139.321611:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:3.0:1788879139.321615:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 02000000:00000001:2.0:1788879139.321616:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879139.321623:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1788879139.321624:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879139.321631:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290133376:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879139.321634:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321637:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.321641:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879139.321644:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00100000:2.0:1788879139.321645:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290133760/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879139.321648:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879139.321653:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879139.321657:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:2.0:1788879139.321659:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000020:00000001:3.0:1788879139.321661:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:2.0:1788879139.321665:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879139.321668:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1788879139.321670:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879139.321671:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879139.321676:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879139.321678:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.321680:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.321686:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879139.321688:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321691:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879139.321693:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879139.321694:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879139.321696:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879139.321697:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879139.321699:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879139.321701:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879139.321704:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879139.321710:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290133504:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879139.321713:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879139.321715:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879139.321716:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879139.321718:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879139.321721:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321726:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879139.321727:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00000001:3.0:1788879139.321732:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879139.321743:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879139.321747:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879139.321749:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879139.321750:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321752:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879139.321754:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321758:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304750603 00000100:00000001:3.0:1788879139.321761:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304751145 00000100:00000001:3.0:1788879139.321763:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304750603 00000100:00000001:3.0:1788879139.321765:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304751145 00000100:00000001:3.0:1788879139.321767:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879139.321769:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321770:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879139.321774:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879139/real 1788879139] req@ffff94e5045e7480 x1875765290133504/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879139.321781:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879139.321782:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879139.321784:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879139.321786:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879139.321787:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.321789:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879139.321790:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879139.321793:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290133504/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879139.321806:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.321808:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879139.321810:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.321811:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879139.321812:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.321814:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879139.321825:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879139.321828:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879139.321830:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879139.321832:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879139.321835:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290133504:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879139.321850:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879139.321852:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879139.321854:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879139.321857:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879139.321859:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879139.321863:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879139.321867:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290133632:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879139.321872:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879139.321874:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879139.321876:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879139.321879:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879139.321883:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321891:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879139.321894:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00000001:3.0:1788879139.321898:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879139.321971:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879139.321986:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879139.321989:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879139.321991:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.321994:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879139.321998:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.322004:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304750602 00000100:00000001:3.0:1788879139.322008:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304751145 00000100:00000001:3.0:1788879139.322011:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304750602 00000100:00000001:3.0:1788879139.322014:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304751145 00000100:00000001:3.0:1788879139.322017:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879139.322019:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.322020:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879139.322025:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879139/real 1788879139] req@ffff94e5045e7b80 x1875765290133632/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879139.322033:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879139.322034:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879139.322047:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879139.322048:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879139.322049:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.322051:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879139.322053:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879139.322056:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290133632/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879139.322062:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.322064:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879139.322066:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.322067:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879139.322068:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.322070:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879139.322079:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879139.322081:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879139.322085:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879139.322087:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879139.322090:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290133632:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879139.322094:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879139.322095:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879139.322096:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879139.322098:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879139.322100:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879139.322102:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879139.322104:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290133760:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879139.322107:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879139.322109:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879139.322110:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879139.322112:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879139.322115:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.322119:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879139.322120:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:3.0:1788879139.322123:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879139.322135:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879139.322138:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879139.322141:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879139.322142:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.322144:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879139.322146:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.322149:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304750602 00000100:00000001:3.0:1788879139.322151:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304751145 00000100:00000001:3.0:1788879139.322153:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304750602 00000100:00000001:3.0:1788879139.322155:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304751145 00000100:00000001:3.0:1788879139.322157:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879139.322158:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.322160:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879139.322163:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879139/real 1788879139] req@ffff94e5045e4e00 x1875765290133760/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879139.322176:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879139.322177:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879139.322179:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879139.322180:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879139.322182:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.322183:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879139.322184:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879139.322187:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290133760/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879139.322193:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.322194:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879139.322196:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.322197:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879139.322198:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.322200:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879139.322226:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879139.322228:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879139.322231:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879139.322233:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879139.322236:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290133760:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879139.322239:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.322241:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879139.322243:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879139.322244:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879139.322246:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879139.322248:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879139.322249:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879139.322251:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879139.322253:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879139.322255:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879139.322256:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.322258:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879139.322259:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879139.322260:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879139.322262:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879139.322264:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879139.322265:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879139.322267:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879139.322268:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879139.322269:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879139.322270:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.322272:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879139.322273:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879139.322274:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879139.322275:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879139.322276:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879139.322277:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879139.322278:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879139.322279:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879139.322281:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879139.322282:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.322283:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879139.322287:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879139.322288:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879139.322290:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879139.322291:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879139.384985:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879139.384993:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879139.384996:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879139.385003:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879139.385009:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879139.385013:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879139.385018:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879139.385021:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879139.385024:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879140.089053:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879140.089061:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879140.089067:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879140.089069:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879140.089092:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879140.089094:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879140.089097:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879140.409041:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879140.409048:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879140.409050:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879140.409055:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879140.409059:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879140.409062:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879140.409065:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879140.409067:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879140.409070:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879141.113050:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879141.113056:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879141.113060:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879141.113062:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879141.113107:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879141.113110:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879141.113112:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879141.432969:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879141.432974:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879141.432976:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879141.432981:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879141.432984:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879141.432987:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879141.432989:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879141.432991:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879141.432993:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879142.137052:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879142.137058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879142.137062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879142.137064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879142.137132:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879142.137137:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879142.137142:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879142.456992:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879142.457000:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879142.457002:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879142.457008:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879142.457011:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879142.457015:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879142.457018:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879142.457020:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879142.457023:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879143.160981:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879143.160986:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879143.160990:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879143.160992:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879143.160994:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879143.160996:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879143.160998:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879143.160999:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879143.161001:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879143.161002:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879143.161035:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879143.161037:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879143.161039:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879143.480964:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879143.480970:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879143.480972:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879143.480976:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879143.480979:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879143.480982:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879143.480984:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879143.480987:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879143.480989:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879144.185002:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879144.185008:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879144.185013:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879144.185014:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879144.185033:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879144.185036:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879144.185038:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879144.441127:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879144.441144:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879144.441148:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879144.441154:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879144.441160:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879144.441169:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10083 00000100:00080000:0.0:1788879144.441182:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879144.441186:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879144.441189:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879144.441218:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879144.441222:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879144.441226:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879144.441230:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879144.441233:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879144.441235:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879144.441240:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879144.441243:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879144.441247:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879144.441251:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879144.441255:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441258:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879144.441287:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879144.441298:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879144.441299:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441302:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879144.441305:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879144.441310:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441322:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879144.441332:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290133888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879144.441343:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879144.441347:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441365:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879144.441369:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879144.441373:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879144.441374:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879144.441376:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879144.441379:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879144.441381:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10083 00000100:00080000:0.0:1788879144.441385:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879144.441387:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879144.441389:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879144.441393:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879144.441395:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879144.441397:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879144.441400:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879144.441402:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879144.441404:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879144.441407:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1788879144.441408:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879144.441415:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.441415:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788879144.441418:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879144.441421:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879144.441425:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788879144.441426:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879144.441428:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879144.441430:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441435:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788879144.441436:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:0.0:1788879144.441440:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788879144.441441:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788879144.441448:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879144.441450:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441455:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879144.441462:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290133888:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788879144.441463:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879144.441467:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441472:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879144.441476:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879144.441477:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879144.441483:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879144.441485:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879144.441489:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441490:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879144.441495:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1788879144.441501:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290134016/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879144.441503:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441515:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879144.441518:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441522:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879144.441525:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879144.441526:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1788879144.441529:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879144.441531:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:0.0:1788879144.441534:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879144.441539:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879144.441539:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879144.441547:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879144.441550:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10083 00000100:00080000:0.0:1788879144.441554:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879144.441556:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879144.441558:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879144.441562:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879144.441563:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879144.441565:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879144.441567:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879144.441570:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879144.441572:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879144.441576:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879144.441578:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879144.441580:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879144.441583:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:3.0:1788879144.441589:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:0.0:1788879144.441590:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441596:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879144.441598:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879144.441602:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441604:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.441605:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:0.0:1788879144.441611:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879144.441612:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1788879144.441614:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.441618:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441619:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879144.441624:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879144.441625:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:0.0:1788879144.441627:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.441633:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441635:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.441639:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304756265 00000100:00100000:0.0:1788879144.441644:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979180 x1875765290134144/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879144.441653:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879144.441656:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788879144.441657:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441683:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879144.441688:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879144.441690:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304756265 00000100:00080000:0.0:1788879144.441700:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879144.441704:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788879144.441705:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879144.441713:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879144.441714:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879144.441721:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879144.441723:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879144.441731:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10083 00000100:00000001:3.0:1788879144.441732:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:0.0:1788879144.441760:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879144.441766:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1788879144.441768:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879144/real 1788879144] req@ffff94e50897bb80 x1875765290133888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879144.441769:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879144.441776:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879144.441778:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879144.441782:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879144.441788:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879144.441796:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879144.441801:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879144.441813:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879144.441818:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879144.441820:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1788879144.441821:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:3.0:1788879144.441830:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1788879144.441831:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879144.441839:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441841:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879144.441845:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879144.441845:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000008:00000001:0.0:1788879144.441853:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441857:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.441857:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879144.441866:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879144.441868:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879144.441875:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1788879144.441876:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441883:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879144.441883:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879144.441888:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879144.441968:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879144.441978:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879144.442008:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290134272/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879144.442018:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290133888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879144.442080:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879144.442082:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879144.442090:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.442091:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879144.442096:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.442098:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442103:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879144.442107:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442111:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879144.442115:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879144.442121:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879144.442132:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879144.442138:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879144.442157:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290133888:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879144.442167:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442171:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879144.442174:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879144.442178:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879144.442184:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879144.442188:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879144.442191:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879144.442197:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879144.442201:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879144.442223:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879144.442225:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442227:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442235:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879144.442237:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.442239:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879144.442241:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879144.442242:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879144.442244:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879144.442245:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879144.442248:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879144.442250:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879144.442252:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879144.442256:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290134016:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879144.442260:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879144.442262:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879144.442264:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879144.442266:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879144.442269:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.442277:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879144.442279:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000100:00000001:3.0:1788879144.442283:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879144.442300:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879144.442305:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879144.442307:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879144.442309:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.442310:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879144.442313:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.442317:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304755724 00000100:00000001:3.0:1788879144.442320:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304756265 00000100:00000001:3.0:1788879144.442323:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304755724 00000100:00000001:3.0:1788879144.442325:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304756265 00000100:00000001:3.0:1788879144.442327:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879144.442329:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.442331:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879144.442335:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879144/real 1788879144] req@ffff94e50897b800 x1875765290134016/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879144.442343:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879144.442345:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879144.442347:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879144.442348:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879144.442350:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442352:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879144.442353:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879144.442357:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290134016/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879144.442364:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442366:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879144.442368:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442369:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879144.442370:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442373:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879144.442384:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879144.442387:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879144.442390:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879144.442392:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879144.442395:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290134016:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879144.442399:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879144.442400:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879144.442402:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879144.442404:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879144.442406:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879144.442407:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879144.442410:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290134144:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879144.442413:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879144.442414:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879144.442416:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879144.442417:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879144.442420:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.442424:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879144.442425:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 00000100:00000001:3.0:1788879144.442429:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879144.442438:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879144.442441:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879144.442443:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879144.442444:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.442446:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879144.442448:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.442451:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304755722 00000100:00000001:3.0:1788879144.442454:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304756265 00000100:00000001:3.0:1788879144.442456:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304755722 00000100:00000001:3.0:1788879144.442458:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304756265 00000100:00000001:3.0:1788879144.442462:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879144.442464:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.442465:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879144.442469:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879144/real 1788879144] req@ffff94e508979180 x1875765290134144/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879144.442476:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879144.442477:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879144.442479:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879144.442480:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879144.442482:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442483:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879144.442485:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879144.442488:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290134144/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879144.442495:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442497:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879144.442499:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442500:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879144.442501:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442503:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879144.442512:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879144.442514:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879144.442517:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879144.442519:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879144.442522:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290134144:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879144.442525:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879144.442526:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879144.442527:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879144.442529:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879144.442531:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879144.442533:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879144.442536:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290134272:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879144.442539:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879144.442540:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879144.442542:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879144.442544:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879144.442546:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.442549:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879144.442550:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 00000100:00000001:3.0:1788879144.442553:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879144.442561:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879144.442565:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879144.442566:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879144.442568:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.442569:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879144.442571:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.442574:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304755722 00000100:00000001:3.0:1788879144.442576:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304756265 00000100:00000001:3.0:1788879144.442578:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304755722 00000100:00000001:3.0:1788879144.442581:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304756265 00000100:00000001:3.0:1788879144.442583:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879144.442584:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.442586:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879144.442589:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879144/real 1788879144] req@ffff94e508978380 x1875765290134272/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879144.442596:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879144.442597:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879144.442599:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879144.442600:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879144.442601:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442603:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879144.442604:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879144.442607:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290134272/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879144.442620:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442622:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879144.442625:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442626:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879144.442627:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442629:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879144.442638:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879144.442640:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879144.442642:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879144.442644:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879144.442647:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290134272:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879144.442651:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442653:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879144.442654:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879144.442656:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879144.442657:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879144.442659:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879144.442660:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879144.442663:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879144.442664:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879144.442666:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879144.442667:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442669:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879144.442670:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879144.442671:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879144.442673:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879144.442674:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879144.442675:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879144.442677:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879144.442678:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879144.442680:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879144.442681:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442682:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879144.442683:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879144.442685:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879144.442686:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879144.442688:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879144.442689:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879144.442691:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879144.442692:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879144.442693:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879144.442695:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442696:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879144.442699:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879144.442700:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879144.442702:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879144.442703:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879144.505038:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879144.505045:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879144.505048:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879144.505054:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879144.505059:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879144.505063:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879144.505067:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879144.505070:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879144.505073:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879145.208992:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879145.208998:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879145.209000:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879145.209002:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879145.209006:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879145.209008:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879145.209009:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879145.209011:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879145.209013:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879145.209014:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879145.209030:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879145.209032:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879145.209034:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879145.528991:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879145.528997:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879145.528998:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879145.529003:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879145.529005:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879145.529009:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879145.529011:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879145.529013:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879145.529015:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879146.233065:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879146.233073:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879146.233079:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879146.233080:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879146.233151:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879146.233155:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879146.233159:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879146.553008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879146.553017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879146.553019:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879146.553026:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879146.553030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879146.553035:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879146.553038:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879146.553041:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879146.553044:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879147.257115:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879147.257125:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879147.257132:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879147.257134:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879147.257224:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879147.257229:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879147.257232:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879147.577042:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879147.577051:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879147.577054:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879147.577060:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879147.577065:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879147.577069:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879147.577073:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879147.577076:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879147.577079:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879148.281054:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879148.281063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879148.281069:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879148.281070:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879148.281108:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879148.281111:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879148.281114:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879148.601024:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879148.601032:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879148.601035:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879148.601041:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879148.601046:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879148.601050:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879148.601053:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879148.601057:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879148.601060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879149.305046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879149.305052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879149.305057:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879149.305058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879149.305108:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879149.305112:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879149.305116:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879149.560987:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879149.560995:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879149.560997:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879149.561000:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879149.561003:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879149.561008:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10088 00000100:00080000:3.0:1788879149.561018:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879149.561020:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879149.561021:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879149.561028:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879149.561030:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879149.561033:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879149.561035:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879149.561037:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879149.561038:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879149.561041:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879149.561043:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879149.561045:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879149.561048:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879149.561061:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879149.561063:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879149.561067:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879149.561078:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879149.561080:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879149.561083:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879149.561086:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879149.561091:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879149.561103:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879149.561114:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290134400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879149.561123:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879149.561127:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879149.561141:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879149.561144:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879149.561146:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879149.561147:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879149.561149:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879149.561151:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879149.561153:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10088 00000100:00080000:3.0:1788879149.561157:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879149.561159:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879149.561160:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879149.561163:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879149.561164:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879149.561166:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879149.561168:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788879149.561168:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879149.561172:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879149.561173:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879149.561174:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879149.561178:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1788879149.561179:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879149.561181:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879149.561183:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561184:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879149.561186:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879149.561189:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879149.561193:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879149.561193:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879149.561197:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879149.561197:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879149.561202:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879149.561206:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879149.561207:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290134400:192.168.203.129@tcp:250:kworker.0 00000008:00000001:3.0:1788879149.561222:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561224:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879149.561226:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879149.561228:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879149.561230:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879149.561232:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879149.561233:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879149.561237:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879149.561237:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879149.561238:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879149.561241:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879149.561244:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879149.561246:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879149.561252:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290134528/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879149.561258:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879149.561260:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879149.561260:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561264:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:3.0:1788879149.561266:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561269:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879149.561269:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879149.561274:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879149.561275:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879149.561276:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879149.561279:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879149.561280:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10088 00000100:00080000:3.0:1788879149.561283:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879149.561284:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879149.561285:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879149.561287:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879149.561288:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879149.561289:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879149.561291:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879149.561293:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879149.561295:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879149.561297:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879149.561299:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879149.561300:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879149.561302:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879149.561303:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879149.561304:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879149.561306:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879149.561307:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561309:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879149.561310:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879149.561316:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879149.561317:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304761385 02000000:00000001:3.0:1788879149.561318:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879149.561322:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879149.561323:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879149.561324:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879149.561328:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561329:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00100000:3.0:1788879149.561331:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290134656/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879149.561337:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879149.561339:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304761385 00000100:00000001:3.0:1788879149.561339:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879149.561343:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879149.561345:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879149.561346:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788879149.561351:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879149.561353:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879149.561354:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879149.561357:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879149.561358:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879149.561361:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879149.561363:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879149.561364:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10088 00000100:00000001:1.0:1788879149.561369:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788879149.561369:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879149.561373:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879149.561375:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879149.561377:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879149.561378:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:3.0:1788879149.561379:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879149.561383:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000001:3.0:1788879149.561383:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879149.561386:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879149.561387:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879149.561389:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879149.561390:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879149.561391:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879149.561393:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1788879149.561396:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879149.561397:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879149.561400:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561401:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879149.561403:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879149.561405:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879149.561407:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879149.561408:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879149.561410:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879149.561412:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879149.561413:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879149.561416:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879149.561420:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290134528:192.168.203.129@tcp:38:kworker.0 00000008:00000001:3.0:1788879149.561420:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879149.561424:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561428:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879149.561430:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879149.561431:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879149.561433:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879149.561433:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879149.561438:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879149.561438:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879149.561442:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879149.561443:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879149.561444:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879149.561449:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561452:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00100000:3.0:1788879149.561455:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290134784/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879149.561456:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:1.0:1788879149.561463:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879149.561466:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879149.561468:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879149.561471:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561476:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561481:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304760843 00000100:00000001:1.0:1788879149.561483:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304761385 00000100:00000001:1.0:1788879149.561485:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304760843 00000100:00000001:1.0:1788879149.561487:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304761385 00000100:00000001:1.0:1788879149.561490:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879149.561491:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561493:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879149.561494:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879149.561495:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879149.561497:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879149.561499:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879149.561501:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879149.561505:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290134656:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879149.561508:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879149.561509:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879149.561510:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879149.561511:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879149.561512:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561516:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879149.561517:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:1.0:1788879149.561519:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879149.561525:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561527:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304760841 00000100:00000001:1.0:1788879149.561528:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304761385 00000100:00000001:1.0:1788879149.561530:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304760841 00000100:00000001:1.0:1788879149.561531:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304761385 00000100:00000001:1.0:1788879149.561532:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879149.561534:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561535:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561536:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879149.561538:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879149.561539:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788879149.561540:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879149.561541:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879149.561542:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879149.561543:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879149.561544:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879149.561545:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879149.561546:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879149.561548:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879149.561550:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290134784:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879149.561551:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879149.561552:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879149.561553:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879149.561554:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879149.561555:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561558:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879149.561559:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:1.0:1788879149.561561:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879149.561566:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561568:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304760841 00000100:00000001:1.0:1788879149.561569:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304761385 00000100:00000001:1.0:1788879149.561571:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304760841 00000100:00000001:1.0:1788879149.561572:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304761385 00000100:00000001:1.0:1788879149.561573:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879149.561575:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561576:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561577:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879149.561578:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879149.561579:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788879149.561580:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879149.561581:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879149.561582:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.561583:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879149.624964:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879149.624969:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879149.624971:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879149.624987:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879149.624990:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879149.624992:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879149.624995:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879149.624997:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879149.624999:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879150.328989:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879150.328994:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879150.328998:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879150.328999:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879150.329059:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879150.329064:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879150.329068:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879150.648984:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879150.648992:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879150.648995:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879150.649001:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879150.649005:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879150.649009:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879150.649012:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879150.649015:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879150.649018:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879151.353022:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879151.353031:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879151.353038:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879151.353039:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879151.353076:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879151.353080:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879151.353084:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879151.673018:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879151.673027:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879151.673029:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879151.673035:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879151.673039:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879151.673043:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879151.673046:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879151.673049:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879151.673052:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879152.377033:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879152.377043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879152.377062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879152.377064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879152.377094:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879152.377098:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879152.377103:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788879152.633410:0:2351:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1788879152.633524:0:2351:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788879152.633569:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879152.633599:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879152.633650:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879152.633657:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879152.633663:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879152.633689:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879152.633704:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879152.633721:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879152.633727:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879152.633731:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879152.633755:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000100:0.0:1788879152.633756:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788879152.633769:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879152.633776:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879152.633783:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879152.633789:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879152.633800:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879152.633807:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:3.0:1788879152.633814:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879149/real 1788879152] req@ffff94e50449d880 x1875765290134528/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1788879152.633828:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879152.633876:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879152.633886:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879152.633894:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879152.633972:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000400:00000001:3.0:1788879152.634041:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879152.634047:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879152.634063:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879152.634070:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879152.634074:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.634081:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879152.634085:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879152.634097:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290134528/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879152.634113:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.634121:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879152.634127:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.634130:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879152.634133:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.634138:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879152.634166:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879152.634173:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879152.634183:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879152.634190:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879152.634199:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290134528:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879152.634208:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879152.634217:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879149/real 1788879152] req@ffff94e50449fb80 x1875765290134656/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879152.634233:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879152.634235:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879152.634242:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879152.634245:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879152.634248:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.634251:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879152.634255:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879152.634262:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290134656/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879152.634274:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.634279:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879152.634283:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.634286:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879152.634289:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.634291:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879152.634315:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879152.634320:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879152.634327:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879152.634332:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879152.634338:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290134656:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879152.634345:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879152.634352:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879149/real 1788879152] req@ffff94e50449c380 x1875765290134400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879152.634367:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879152.634369:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879152.634375:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879152.634378:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879152.634380:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.634384:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879152.634386:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879152.634393:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290134400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879152.636054:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.636061:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879152.636066:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.636071:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879152.636075:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.636079:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879152.636082:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879152.636088:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879152.636100:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879152.636104:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879152.636111:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290134400:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879152.636122:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879152.636127:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879152.636130:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879152.636134:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879152.636143:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879152.636262:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879152.636266:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879152.636270:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879152.636272:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879152.636276:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879152.636278:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.636280:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879152.636282:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879152.636284:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879152.636287:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879152.636288:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879152.636289:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879152.636291:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879152.636293:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879152.636294:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879152.636296:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.636298:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879152.636299:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879152.636300:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879152.636302:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879152.636303:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879152.636305:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879152.636306:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879152.636307:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879152.636309:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879152.636311:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.636314:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879152.636317:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879152.636319:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879152.636322:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879152.636331:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879149/real 1788879152] req@ffff94e50449d180 x1875765290134784/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879152.636347:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879152.636349:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879152.636356:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879152.636359:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879152.636361:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.636363:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879152.636364:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879152.636369:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290134784/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879152.636377:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.636380:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879152.636382:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.636384:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879152.636385:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.636387:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879152.636399:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879152.636402:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879152.636406:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879152.636408:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879152.636411:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290134784:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879152.636416:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.636418:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879152.636419:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879152.636421:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879152.636423:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879152.636425:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879152.636426:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879152.636428:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879152.636429:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879152.636431:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879152.636432:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.636434:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879152.636442:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879152.636444:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879152.636446:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879152.636447:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879152.696995:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879152.697003:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879152.697005:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879152.697010:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879152.697014:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879152.697018:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879152.697021:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879152.697024:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879152.697027:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879153.401032:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879153.401039:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879153.401045:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879153.401050:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879153.401053:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879153.401057:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879153.401060:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879153.401062:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879153.401065:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879153.401068:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879153.401121:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879153.401125:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879153.401129:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879153.721039:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879153.721051:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879153.721054:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879153.721062:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879153.721068:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879153.721073:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879153.721077:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879153.721082:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879153.721086:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879154.425024:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879154.425031:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879154.425036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879154.425037:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879154.425120:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879154.425126:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879154.425131:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879154.681032:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879154.681046:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879154.681049:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879154.681054:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879154.681059:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879154.681069:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10093 00000100:00080000:3.0:1788879154.681087:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879154.681093:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879154.681098:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879154.681112:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879154.681116:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879154.681122:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879154.681127:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879154.681131:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879154.681134:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879154.681143:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879154.681149:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879154.681155:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879154.681162:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879154.681167:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.681173:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879154.681178:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879154.681192:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879154.681194:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.681198:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879154.681202:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879154.681208:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.681222:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879154.681235:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290134912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879154.681250:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879154.681256:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.681285:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879154.681289:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879154.681294:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879154.681296:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879154.681298:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879154.681302:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879154.681306:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10093 00000100:00080000:3.0:1788879154.681312:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879154.681315:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879154.681317:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879154.681322:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879154.681324:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879154.681326:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879154.681329:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879154.681332:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879154.681334:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879154.681338:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879154.681341:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879154.681343:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879154.681351:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879154.681353:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.681356:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879154.681361:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879154.681366:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.681369:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879154.681374:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879154.681376:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.681379:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879154.681382:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879154.681387:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.681397:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.681402:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1788879154.681411:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290135040/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879154.681412:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879154.681419:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879154.681426:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879154.681434:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.681444:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879154.681448:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879154.681456:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879154.681459:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879154.681467:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879154.681468:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879154.681481:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788879154.681489:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290134912:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1788879154.681494:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879154.681502:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879154.681506:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788879154.681510:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1788879154.681517:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788879154.681521:0:178134:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:1.0:1788879154.681526:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879154.681535:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10093 02000000:00000001:1.0:1788879154.681537:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788879154.681550:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788879154.681551:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.681559:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879154.681565:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879154.681575:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879154.681578:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879154.681581:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879154.681584:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1788879154.681587:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879154.681592:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:3.0:1788879154.681597:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879154.681603:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879154.681605:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879154.681619:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879154.681624:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879154.681630:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879154.681636:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879154.681641:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.681646:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879154.681652:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879154.681657:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.681661:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879154.681671:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879154.681678:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879154.681685:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.681691:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879154.681695:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788879154.681696:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879154.681706:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.681708:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879154.681715:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.681720:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.681721:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879154.681734:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879154.681741:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290135168/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879154.681746:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304766505 00000100:00000001:1.0:1788879154.681767:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879154.681783:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879154.681789:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.681794:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879154.681799:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879154.681806:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879154.681810:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879154.681812:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304766505 00000100:00080000:3.0:1788879154.681815:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879154.681855:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879154.681859:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788879154.681865:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10093 00000100:00000001:1.0:1788879154.681871:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879154.681880:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879154.681881:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.681893:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879154.681895:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879154.681946:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:1.0:1788879154.681956:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879154/real 1788879154] req@ffff94e50449d180 x1875765290134912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788879154.681960:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879154.681967:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879154.681975:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879154.681985:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879154.681994:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000400:00000001:1.0:1788879154.681997:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788879154.681998:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:1.0:1788879154.682005:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1788879154.682014:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879154.682018:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879154.682021:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.682026:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:3.0:1788879154.682030:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879154.682034:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879154.682040:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879154.682043:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879154.682050:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879154.682052:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.682061:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879154.682068:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879154.682070:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290134912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1788879154.682079:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879154.682089:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879154.682106:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879154.682107:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879154.682113:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.682117:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879154.682122:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879154.682128:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879154.682130:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879154.682137:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879154.682141:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.682145:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879154.682149:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879154.682155:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879154.682163:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879154.682168:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879154.682202:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290134912:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879154.682214:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.682218:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879154.682220:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879154.682225:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879154.682233:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879154.682236:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879154.682240:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879154.682246:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879154.682249:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879154.682254:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879154.682257:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.682260:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.682272:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879154.682276:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.682280:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879154.682283:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879154.682286:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879154.682289:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879154.682291:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879154.682296:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879154.682300:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879154.682304:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879154.682312:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290135040:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879154.682319:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879154.682323:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879154.682326:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879154.682330:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879154.682337:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.682350:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879154.682353:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:1.0:1788879154.682361:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879154.682391:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879154.682400:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879154.682405:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879154.682408:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.682411:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879154.682417:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.682428:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304765964 00000100:00000001:1.0:1788879154.682433:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304766505 00000100:00000001:1.0:1788879154.682438:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304765964 00000100:00000001:1.0:1788879154.682442:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304766505 00000100:00000001:1.0:1788879154.682446:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879154.682450:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.682454:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879154.682462:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879154/real 1788879154] req@ffff94e50449c380 x1875765290135040/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879154.682480:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879154.682482:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879154.682487:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879154.682491:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879154.682493:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.682496:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879154.682500:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879154.682507:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290135040/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879154.682522:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.682527:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879154.682532:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.682535:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879154.682538:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.682542:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879154.682565:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879154.682571:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879154.682577:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879154.682581:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879154.682587:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290135040:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879154.682594:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879154.682596:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879154.682599:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879154.682603:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879154.682606:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879154.682610:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879154.682616:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290135168:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879154.682623:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879154.682626:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879154.682629:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879154.682632:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879154.682639:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.682652:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788879154.682656:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.682659:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:1.0:1788879154.682670:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879154.682678:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879154.682697:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290135296/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788879154.682707:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879154.682721:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879154.682726:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879154.682730:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879154.682739:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.682740:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.682749:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879154.682751:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.682758:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.682769:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304765962 00000100:00000001:1.0:1788879154.682776:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304766505 00000100:00000001:1.0:1788879154.682781:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304765962 00000100:00000001:1.0:1788879154.682786:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304766505 00000100:00000001:1.0:1788879154.682792:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879154.682796:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.682800:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879154.682810:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879154/real 1788879154] req@ffff94e50449fb80 x1875765290135168/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879154.682847:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879154.682849:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879154.682855:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879154.682859:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879154.682862:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.682866:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879154.682872:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879154.682889:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290135168/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879154.682937:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.682942:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879154.682946:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.682949:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879154.682951:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.682954:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879154.682976:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879154.682981:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879154.682988:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879154.682991:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879154.682997:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290135168:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879154.683004:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.683007:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879154.683010:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879154.683013:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879154.683018:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879154.683021:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879154.683024:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879154.683029:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879154.683031:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879154.683034:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879154.683037:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.683039:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879154.683041:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879154.683043:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879154.683047:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879154.683049:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879154.683051:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879154.683054:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879154.683056:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879154.683059:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879154.683061:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.683065:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.683072:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879154.683076:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.683080:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879154.683083:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879154.683086:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879154.683088:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879154.683091:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879154.683096:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879154.683099:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879154.683103:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879154.683109:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290135296:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879154.683115:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879154.683119:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879154.683122:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879154.683125:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879154.683131:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.683141:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879154.683144:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:1.0:1788879154.683150:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879154.683175:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879154.683195:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879154.683200:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879154.683203:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.683207:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879154.683213:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.683221:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304765963 00000100:00000001:1.0:1788879154.683226:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304766505 00000100:00000001:1.0:1788879154.683230:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304765963 00000100:00000001:1.0:1788879154.683235:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304766505 00000100:00000001:1.0:1788879154.683239:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879154.683244:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.683248:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879154.683257:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879154/real 1788879154] req@ffff94e50449d880 x1875765290135296/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879154.683274:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879154.683276:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879154.683282:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879154.683285:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879154.683288:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.683291:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879154.683294:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879154.683302:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290135296/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879154.683313:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.683317:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879154.683321:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.683324:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879154.683327:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.683330:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879154.683352:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879154.683356:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879154.683362:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879154.683366:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879154.683372:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290135296:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879154.683378:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.683382:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879154.683384:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879154.683387:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879154.683391:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879154.683393:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879154.683396:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879154.683400:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879154.683402:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879154.683405:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879154.683407:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.683410:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.683416:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879154.683419:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.683422:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879154.683425:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879154.745043:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879154.745054:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879154.745058:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879154.745067:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879154.745074:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879154.745081:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.745086:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879154.745090:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879154.745093:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879155.449042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879155.449055:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879155.449060:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879155.449066:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879155.449074:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879155.449078:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879155.449082:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879155.449087:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879155.449095:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879155.449097:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879155.449127:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879155.449131:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879155.449135:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879155.769005:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879155.769014:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879155.769017:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879155.769024:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879155.769029:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879155.769033:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879155.769037:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879155.769041:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879155.769044:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879156.473016:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879156.473023:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879156.473028:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879156.473030:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879156.473059:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879156.473062:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879156.473065:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879156.793070:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879156.793084:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879156.793089:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879156.793102:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879156.793110:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879156.793118:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879156.793123:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879156.793128:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879156.793132:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879157.497017:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879157.497024:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879157.497029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879157.497031:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879157.497065:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879157.497069:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879157.497073:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879157.817033:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879157.817043:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879157.817046:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879157.817054:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879157.817060:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879157.817065:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879157.817069:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879157.817073:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879157.817077:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879158.521103:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879158.521115:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879158.521124:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879158.521126:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879158.521241:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879158.521248:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879158.521253:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879158.841069:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879158.841084:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879158.841089:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879158.841097:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879158.841102:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879158.841108:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879158.841112:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879158.841115:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879158.841118:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879159.545057:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879159.545065:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879159.545070:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879159.545071:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879159.545107:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879159.545110:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879159.545130:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879159.801044:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879159.801057:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879159.801061:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879159.801068:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879159.801084:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879159.801097:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10098 00000100:00080000:3.0:1788879159.801112:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879159.801118:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879159.801122:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879159.801133:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879159.801136:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879159.801140:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879159.801144:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879159.801147:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879159.801149:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879159.801154:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879159.801158:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879159.801162:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879159.801166:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879159.801169:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879159.801172:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879159.801175:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879159.801186:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879159.801189:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879159.801193:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879159.801196:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879159.801201:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879159.801215:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879159.801225:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290135424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879159.801239:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879159.801244:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879159.801267:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879159.801272:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879159.801277:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879159.801280:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879159.801282:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879159.801287:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879159.801292:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10098 00000100:00080000:3.0:1788879159.801298:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879159.801301:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879159.801304:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879159.801307:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879159.801309:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879159.801312:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879159.801317:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788879159.801317:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879159.801324:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879159.801324:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879159.801328:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879159.801348:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879159.801354:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1788879159.801357:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879159.801359:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879159.801364:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879159.801368:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879159.801370:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879159.801376:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879159.801381:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879159.801383:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879159.801393:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290135424:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879159.801400:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879159.801405:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879159.801408:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879159.801411:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879159.801418:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.801435:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879159.801437:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:2.0:1788879159.801442:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879159.801484:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879159.801515:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879159.801520:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879159.801522:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.801525:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879159.801531:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879159.801533:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.801538:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304771625 00000100:00000001:3.0:1788879159.801538:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879159.801542:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879159.801548:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.801551:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879159.801552:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.801563:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304771625 02000000:00000001:3.0:1788879159.801565:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879159.801569:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879159.801573:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879159.801575:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1788879159.801576:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879159.801583:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879159.801584:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.801592:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.801597:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879159.801599:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879159.801607:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879159/real 1788879159] req@ffff94e50449f800 x1875765290135424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879159.801614:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290135552/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1788879159.801641:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788879159.801643:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:2.0:1788879159.801645:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879159.801650:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.801653:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879159.801658:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.801660:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879159.801665:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.801668:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879159.801670:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879159.801676:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290135424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879159.801684:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.801687:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879159.801690:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.801692:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879159.801693:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.801696:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879159.801698:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879159.801701:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879159.801705:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879159.801708:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879159.801712:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290135424:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879159.801716:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.801719:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879159.801720:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879159.801723:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879159.801725:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879159.801728:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879159.801730:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879159.801733:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879159.801735:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879159.801738:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879159.801739:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.801741:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.801750:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879159.801751:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.801754:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879159.801755:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879159.801757:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879159.801759:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879159.801761:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879159.801764:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879159.801766:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879159.801769:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879159.801772:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290135552:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879159.801775:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879159.801777:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879159.801779:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879159.801782:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879159.801811:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.801823:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879159.801825:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00080000:3.0:1788879159.801829:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879159.801833:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879159.801846:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879159.801849:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879159.801852:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879159.801857:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000800:00000100:2.0:1788879159.801864:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1788879159.801866:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10098 00000100:00080000:3.0:1788879159.801877:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879159.801879:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879159.801887:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879159.801891:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.801894:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879159.801936:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.801943:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304771084 00000100:00000001:2.0:1788879159.801947:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304771625 00000100:00000001:2.0:1788879159.801951:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304771084 00000100:00000001:2.0:1788879159.801955:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304771625 00000100:00000001:2.0:1788879159.801958:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879159.801960:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.801963:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879159.801972:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879159/real 1788879159] req@ffff94e50449f480 x1875765290135552/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879159.801984:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879159.801987:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879159.801989:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879159.801994:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879159.801995:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879159.802002:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879159.802009:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788879159.802009:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879159.802016:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1788879159.802018:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879159.802019:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788879159.802026:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00100000:2.0:1788879159.802030:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290135552/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788879159.802036:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879159.802045:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879159.802047:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879159.802050:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879159.802056:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879159.802056:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879159.802061:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879159.802066:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802070:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1788879159.802074:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879159.802083:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802087:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788879159.802087:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879159.802092:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788879159.802093:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1788879159.802101:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879159.802107:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879159.802108:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879159.802116:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00100000:3.0:1788879159.802117:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879159.802126:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879159.802127:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290135552:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879159.802132:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802154:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802159:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788879159.802161:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879159.802163:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879159.802168:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879159.802170:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788879159.802178:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879159.802179:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:3.0:1788879159.802182:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:2.0:1788879159.802184:0:2361:0:(genops.c:1216:class_import_put()) Process entered 02000000:00000001:3.0:1788879159.802189:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879159.802190:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879159.802197:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879159.802218:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879159.802222:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879159.802222:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802226:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802230:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802235:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1788879159.802236:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290135680/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879159.802239:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802246:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879159.802250:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879159.802267:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879159.802270:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879159.802280:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879159.802283:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879159.802287:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879159.802290:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879159.802293:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879159.802296:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879159.802300:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1788879159.802301:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10098 00000100:00000001:2.0:1788879159.802306:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788879159.802310:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879159.802311:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879159.802315:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879159.802318:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879159.802319:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879159.802322:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879159.802325:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1788879159.802330:0:178134:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1788879159.802331:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788879159.802336:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:2.0:1788879159.802337:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879159.802340:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879159.802347:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:2.0:1788879159.802348:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290135680:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879159.802355:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879159.802358:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879159.802360:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879159.802362:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879159.802366:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879159.802369:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:2.0:1788879159.802371:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879159.802377:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879159.802378:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879159.802381:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879159.802387:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879159.802391:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788879159.802392:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802397:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:3.0:1788879159.802398:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879159.802404:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879159.802407:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:3.0:1788879159.802410:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879159.802415:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879159.802421:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879159.802423:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879159.802426:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879159.802428:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000800:00000100:2.0:1788879159.802433:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879159.802434:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802443:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879159.802445:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802449:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879159.802454:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879159.802456:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290135808/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879159.802458:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879159.802464:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802471:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304771082 00000100:00000001:3.0:1788879159.802477:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879159.802479:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304771625 00000100:00000001:3.0:1788879159.802482:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802486:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304771082 00000100:00000001:3.0:1788879159.802490:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802491:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304771625 00000100:00000001:2.0:1788879159.802498:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879159.802501:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802503:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879159.802509:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879159/real 1788879159] req@ffff94e50449c700 x1875765290135680/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879159.802518:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879159.802519:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879159.802523:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879159.802526:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879159.802528:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802531:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879159.802533:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879159.802539:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290135680/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879159.802547:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802549:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879159.802551:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802553:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879159.802554:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802556:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879159.802567:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879159.802570:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879159.802574:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879159.802576:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879159.802579:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290135680:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879159.802582:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802584:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879159.802585:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879159.802587:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879159.802589:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879159.802590:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879159.802591:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879159.802593:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879159.802594:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879159.802596:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879159.802598:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802600:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802603:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879159.802604:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802606:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879159.802608:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879159.802609:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879159.802611:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879159.802613:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879159.802615:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879159.802617:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879159.802619:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879159.802622:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290135808:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879159.802624:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879159.802626:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879159.802627:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879159.802629:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879159.802632:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802636:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879159.802637:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:2.0:1788879159.802641:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879159.802651:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879159.802654:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879159.802656:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879159.802658:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802659:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879159.802662:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802665:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304771082 00000100:00000001:2.0:1788879159.802668:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304771625 00000100:00000001:2.0:1788879159.802670:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304771082 00000100:00000001:2.0:1788879159.802672:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304771625 00000100:00000001:2.0:1788879159.802674:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879159.802676:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802678:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879159.802682:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879159/real 1788879159] req@ffff94e50449e300 x1875765290135808/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879159.802689:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879159.802690:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879159.802692:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879159.802694:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879159.802695:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802698:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879159.802699:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879159.802703:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290135808/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879159.802710:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802712:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879159.802714:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802716:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879159.802717:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802718:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879159.802729:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879159.802731:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879159.802734:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879159.802736:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879159.802739:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290135808:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879159.802742:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802743:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879159.802745:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879159.802746:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879159.802748:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879159.802749:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879159.802750:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879159.802752:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879159.802754:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879159.802755:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879159.802757:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802758:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879159.802761:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879159.802762:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879159.802764:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879159.802766:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879159.864967:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879159.864972:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879159.864974:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879159.864978:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879159.864981:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879159.864984:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879159.864985:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879159.864987:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879159.864990:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879160.569002:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879160.569010:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879160.569017:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879160.569019:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879160.569045:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879160.569049:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879160.569053:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879160.889034:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879160.889043:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879160.889045:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879160.889052:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879160.889057:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879160.889062:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879160.889066:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879160.889069:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879160.889072:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879161.593038:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879161.593045:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879161.593052:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879161.593054:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879161.593123:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879161.593127:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879161.593132:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879161.912978:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879161.912986:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879161.912988:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879161.912994:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879161.912998:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879161.913002:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879161.913005:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879161.913008:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879161.913011:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879162.617042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879162.617051:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879162.617060:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879162.617062:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879162.617139:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879162.617144:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879162.617148:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879162.937004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879162.937012:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879162.937014:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879162.937021:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879162.937025:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879162.937030:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879162.937033:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879162.937035:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879162.937038:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879163.641034:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879163.641042:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879163.641049:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879163.641053:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879163.641056:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879163.641060:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879163.641064:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879163.641066:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879163.641069:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879163.641072:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879163.641109:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879163.641112:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879163.641116:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879163.961027:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879163.961038:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879163.961043:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879163.961052:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879163.961059:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879163.961065:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879163.961070:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879163.961074:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879163.961099:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879164.665051:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879164.665058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879164.665064:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879164.665065:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879164.665089:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879164.665092:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879164.665094:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879164.921003:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879164.921016:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879164.921019:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879164.921024:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879164.921030:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879164.921039:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10103 00000100:00080000:3.0:1788879164.921052:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879164.921056:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879164.921057:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879164.921074:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879164.921078:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879164.921081:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879164.921084:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879164.921086:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879164.921087:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879164.921091:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879164.921093:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879164.921097:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879164.921101:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879164.921103:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879164.921107:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879164.921109:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879164.921123:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879164.921125:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879164.921128:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879164.921131:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879164.921137:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879164.921152:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879164.921162:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290135936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879164.921172:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879164.921176:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879164.921193:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879164.921195:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879164.921198:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879164.921200:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879164.921201:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879164.921203:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879164.921205:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10103 00000100:00080000:3.0:1788879164.921207:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879164.921209:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879164.921210:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879164.921212:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879164.921212:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879164.921215:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879164.921217:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879164.921220:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879164.921221:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879164.921224:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879164.921226:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879164.921228:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879164.921236:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879164.921259:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.921260:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879164.921270:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879164.921272:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879164.921279:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1788879164.921280:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879164.921286:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:3.0:1788879164.921289:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.921292:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879164.921295:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879164.921309:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879164.921313:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879164.921316:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879164.921321:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879164.921324:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879164.921337:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1788879164.921348:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290135936:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788879164.921349:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879164.921359:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.921361:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879164.921369:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879164.921374:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879164.921379:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879164.921380:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879164.921391:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879164.921416:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290136064/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879164.921418:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879164.921424:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:2.0:1788879164.921450:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879164.921463:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879164.921466:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879164.921472:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879164.921475:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879164.921479:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879164.921480:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879164.921482:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879164.921487:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879164.921490:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10103 00000100:00080000:3.0:1788879164.921496:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879164.921499:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879164.921500:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:2.0:1788879164.921510:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:3.0:1788879164.921513:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879164.921516:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879164.921519:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879164.921541:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:3.0:1788879164.921542:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879164.921553:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879164.921556:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879164.921556:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879164.921568:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.921579:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:3.0:1788879164.921579:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879164.921590:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879164.921594:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879164.921595:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.921608:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304776745 02000000:00000001:3.0:1788879164.921609:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879164.921622:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879164.921627:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879164.921632:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00100000:3.0:1788879164.921633:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879164.921643:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879164.921646:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879164.921653:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879164.921655:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879164.921659:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879164.921661:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304776745 02000000:00000001:3.0:1788879164.921664:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879164.921671:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1788879164.921674:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.921679:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879164.921685:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.921688:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879164.921694:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879164.921703:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879164/real 1788879164] req@ffff94e50449f100 x1875765290135936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879164.921720:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290136192/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1788879164.921726:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879164.921733:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879164.921743:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879164.921776:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879164.921781:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.921799:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879164.921802:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879164.921802:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879164.921813:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879164.921817:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290135936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879164.921820:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879164.921834:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879164.921856:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879164.921863:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879164.921866:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788879164.921868:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879164.921873:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879164.921874:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879164.921885:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10103 00000100:00000001:2.0:1788879164.921886:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.921940:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1788879164.921943:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879164.921945:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879164.921948:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879164.921951:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879164.921952:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879164.921959:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1788879164.921962:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879164.921970:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00080000:2.0:1788879164.921972:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:3.0:1788879164.921977:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00080000:2.0:1788879164.921984:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000020:00000001:3.0:1788879164.921985:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879164.921991:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788879164.921993:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879164.922002:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:2.0:1788879164.922011:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290135936:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1788879164.922016:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879164.922030:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.922032:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879164.922034:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879164.922037:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788879164.922041:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879164.922042:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879164.922050:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879164.922051:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879164.922061:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:2.0:1788879164.922065:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000008:00100000:3.0:1788879164.922066:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879164.922071:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879164.922072:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879164.922080:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788879164.922082:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879164.922092:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879164.922095:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879164.922096:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:3.0:1788879164.922102:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879164.922106:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879164.922108:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879164.922120:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879164.922120:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879164.922127:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879164.922129:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879164.922149:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.922152:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879164.922156:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879164.922162:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290136320/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879164.922164:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879164.922170:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879164.922177:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879164.922185:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879164.922185:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879164.922190:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879164.922192:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879164.922201:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879164.922207:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879164.922209:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879164.922222:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879164.922240:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290136064:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879164.922246:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879164.922250:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879164.922252:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879164.922255:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879164.922259:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.922269:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879164.922272:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:2.0:1788879164.922279:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879164.922302:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879164.922308:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879164.922311:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879164.922313:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.922315:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879164.922319:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.922324:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304776204 00000100:00000001:2.0:1788879164.922328:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304776745 00000100:00000001:2.0:1788879164.922332:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304776204 00000100:00000001:2.0:1788879164.922334:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304776745 00000100:00000001:2.0:1788879164.922337:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879164.922339:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.922342:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879164.922347:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879164/real 1788879164] req@ffff94e50449ca80 x1875765290136064/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879164.922358:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879164.922359:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879164.922362:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879164.922364:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879164.922366:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922369:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879164.922370:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879164.922376:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290136064/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879164.922385:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922388:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879164.922391:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922393:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879164.922394:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922396:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879164.922412:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879164.922415:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879164.922418:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879164.922421:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879164.922425:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290136064:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879164.922429:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879164.922432:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879164.922434:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879164.922437:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879164.922439:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879164.922442:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879164.922445:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290136192:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879164.922449:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879164.922451:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879164.922454:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879164.922456:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879164.922459:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.922464:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879164.922466:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:2.0:1788879164.922480:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879164.922492:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879164.922496:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879164.922499:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879164.922501:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.922503:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879164.922506:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.922510:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304776202 00000100:00000001:2.0:1788879164.922513:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304776745 00000100:00000001:2.0:1788879164.922516:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304776202 00000100:00000001:2.0:1788879164.922518:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304776745 00000100:00000001:2.0:1788879164.922521:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879164.922523:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.922525:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879164.922529:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879164/real 1788879164] req@ffff94e50449c000 x1875765290136192/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879164.922538:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879164.922540:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879164.922542:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879164.922544:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879164.922546:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922548:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879164.922549:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879164.922553:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290136192/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879164.922572:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922575:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879164.922577:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922580:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879164.922581:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922583:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879164.922595:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879164.922598:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879164.922601:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879164.922603:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879164.922608:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290136192:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879164.922613:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922616:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879164.922617:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879164.922619:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879164.922622:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879164.922623:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879164.922625:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879164.922628:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879164.922629:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879164.922632:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879164.922633:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922635:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879164.922637:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879164.922639:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879164.922641:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879164.922643:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879164.922644:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879164.922646:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879164.922647:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879164.922649:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879164.922650:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922652:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922656:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879164.922658:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.922661:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879164.922663:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879164.922665:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879164.922667:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879164.922668:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879164.922671:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879164.922673:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879164.922675:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879164.922679:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290136320:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879164.922683:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879164.922685:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879164.922687:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879164.922689:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879164.922693:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.922698:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879164.922700:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:2.0:1788879164.922703:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879164.922715:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879164.922719:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879164.922722:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879164.922724:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.922726:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879164.922728:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.922733:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304776202 00000100:00000001:2.0:1788879164.922736:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304776745 00000100:00000001:2.0:1788879164.922739:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304776202 00000100:00000001:2.0:1788879164.922742:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304776745 00000100:00000001:2.0:1788879164.922744:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879164.922746:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.922748:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879164.922776:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879164/real 1788879164] req@ffff94e50449e300 x1875765290136320/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879164.922786:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879164.922789:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879164.922792:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879164.922794:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879164.922796:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922799:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879164.922801:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879164.922805:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290136320/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879164.922813:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922816:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879164.922819:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922820:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879164.922823:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922825:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879164.922837:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879164.922840:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879164.922844:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879164.922847:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879164.922851:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290136320:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879164.922855:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922857:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879164.922859:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879164.922860:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879164.922863:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879164.922864:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879164.922866:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879164.922868:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879164.922870:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879164.922872:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879164.922874:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922876:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879164.922879:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879164.922881:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879164.922883:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879164.922885:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879164.984991:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879164.984998:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879164.985000:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879164.985006:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879164.985009:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879164.985013:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879164.985016:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879164.985020:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879164.985022:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879165.689068:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879165.689080:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879165.689083:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879165.689088:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879165.689094:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879165.689096:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879165.689098:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879165.689101:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879165.689105:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879165.689106:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879165.689136:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879165.689140:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879165.689143:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879166.009011:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879166.009023:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879166.009026:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879166.009033:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879166.009039:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879166.009044:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879166.009048:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879166.009052:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879166.009055:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879166.713114:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879166.713123:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879166.713129:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879166.713131:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879166.713222:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879166.713230:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879166.713235:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879167.033020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879167.033026:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879167.033028:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879167.033033:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879167.033036:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879167.033039:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879167.033041:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879167.033043:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879167.033045:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879167.737045:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879167.737054:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879167.737062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879167.737064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879167.737147:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879167.737153:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879167.737159:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879168.056989:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879168.056998:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879168.057001:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879168.057007:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879168.057012:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879168.057018:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879168.057021:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879168.057025:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879168.057028:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879168.761072:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879168.761080:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879168.761085:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879168.761086:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879168.761129:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879168.761131:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879168.761141:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879169.080996:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879169.081002:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879169.081003:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879169.081007:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879169.081009:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879169.081013:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879169.081014:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879169.081016:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879169.081018:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879169.785054:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879169.785063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879169.785071:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879169.785072:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879169.785172:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879169.785176:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879169.785181:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879170.041068:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879170.041079:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879170.041082:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879170.041086:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879170.041091:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879170.041099:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10108 00000100:00080000:0.0:1788879170.041111:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879170.041114:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879170.041117:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879170.041126:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879170.041129:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879170.041132:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879170.041134:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879170.041135:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879170.041136:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879170.041140:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879170.041142:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879170.041144:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879170.041148:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879170.041150:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879170.041152:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879170.041154:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879170.041169:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879170.041170:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879170.041174:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879170.041176:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879170.041182:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879170.041196:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879170.041205:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290136448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879170.041214:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879170.041218:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879170.041238:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879170.041241:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879170.041244:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879170.041245:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879170.041246:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879170.041248:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879170.041250:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10108 00000100:00080000:0.0:1788879170.041253:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879170.041255:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879170.041256:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879170.041258:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879170.041259:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879170.041261:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879170.041263:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879170.041265:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879170.041266:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879170.041269:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879170.041270:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879170.041271:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879170.041277:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879170.041279:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879170.041280:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879170.041284:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:0.0:1788879170.041286:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879170.041289:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041290:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879170.041291:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041295:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1788879170.041299:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879170.041300:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788879170.041302:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041304:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879170.041306:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879170.041308:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879170.041309:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879170.041310:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879170.041315:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879170.041317:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879170.041318:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879170.041327:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290136448:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1788879170.041328:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979180 x1875765290136576/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879170.041337:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788879170.041341:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788879170.041343:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788879170.041346:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879170.041347:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879170.041352:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879170.041353:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879170.041357:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879170.041358:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879170.041364:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879170.041366:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879170.041368:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879170.041372:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879170.041376:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10108 00000100:00000001:2.0:1788879170.041377:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1788879170.041381:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879170.041382:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 00000100:00000001:0.0:1788879170.041388:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879170.041388:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879170.041392:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879170.041397:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879170.041398:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879170.041400:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879170.041402:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879170.041404:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879170.041406:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879170.041408:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879170.041410:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879170.041411:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879170.041414:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879170.041416:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879170.041417:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879170.041420:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879170.041422:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879170.041423:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879170.041428:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879170.041429:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879170.041430:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879170.041431:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879170.041435:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788879170.041435:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879170.041443:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879170.041449:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290136704/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879170.041453:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879170.041455:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879170.041458:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879170.041459:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041463:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879170.041464:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041466:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:0.0:1788879170.041468:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879170.041472:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879170.041473:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879170.041474:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879170.041477:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879170.041479:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304781865 00000100:00000001:0.0:1788879170.041480:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879170.041484:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10108 00000100:00080000:0.0:1788879170.041487:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879170.041489:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879170.041490:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879170.041492:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879170.041493:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1788879170.041494:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879170.041497:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879170.041500:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879170.041502:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879170.041503:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879170.041504:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304781865 00000100:00080000:0.0:1788879170.041507:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879170.041510:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879170.041512:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879170.041512:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1788879170.041517:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879170.041518:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879170.041519:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041522:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879170.041523:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879170.041527:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000008:00100000:0.0:1788879170.041528:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879170.041532:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879170.041534:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879170.041537:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879170/real 1788879170] req@ffff94e508978380 x1875765290136448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788879170.041538:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879170.041541:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879170.041546:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879170.041548:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879170.041550:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879170.041556:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879170.041557:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879170.041560:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:0.0:1788879170.041564:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290136832/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879170.041566:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879170.041570:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879170.041572:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041577:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879170.041581:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879170.041583:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1788879170.041588:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290136448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879170.041589:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879170.041595:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041602:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041605:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879170.041608:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041609:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879170.041611:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041613:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879170.041614:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879170.041616:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879170.041619:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879170.041621:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879170.041624:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290136448:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879170.041628:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041630:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879170.041631:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879170.041633:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879170.041636:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879170.041637:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879170.041639:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879170.041643:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879170.041644:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879170.041646:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879170.041647:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041649:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041656:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879170.041657:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041659:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879170.041660:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879170.041661:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879170.041662:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879170.041663:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879170.041665:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879170.041666:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879170.041667:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879170.041669:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290136576:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879170.041671:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879170.041673:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879170.041674:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879170.041675:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879170.041677:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041682:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879170.041683:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 00000100:00000001:2.0:1788879170.041687:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879170.041695:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879170.041698:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879170.041699:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879170.041700:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041701:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879170.041703:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041706:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304781323 00000100:00000001:2.0:1788879170.041709:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304781865 00000100:00000001:2.0:1788879170.041710:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304781323 00000100:00000001:2.0:1788879170.041712:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304781865 00000100:00000001:2.0:1788879170.041714:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879170.041730:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041731:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879170.041734:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879170/real 1788879170] req@ffff94e508979180 x1875765290136576/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879170.041740:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879170.041741:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879170.041743:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879170.041745:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879170.041746:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041747:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879170.041748:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879170.041750:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290136576/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879170.041755:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041757:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879170.041759:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041760:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879170.041760:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041762:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879170.041776:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879170.041778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879170.041780:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879170.041782:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879170.041784:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290136576:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879170.041787:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879170.041787:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879170.041788:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879170.041790:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879170.041791:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879170.041793:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879170.041794:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290136704:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879170.041797:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879170.041798:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879170.041799:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879170.041800:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879170.041803:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041807:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879170.041808:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000100:00000001:2.0:1788879170.041811:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879170.041819:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879170.041822:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879170.041824:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879170.041825:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041826:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879170.041828:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041830:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304781321 00000100:00000001:2.0:1788879170.041832:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304781865 00000100:00000001:2.0:1788879170.041833:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304781321 00000100:00000001:2.0:1788879170.041835:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304781865 00000100:00000001:2.0:1788879170.041838:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879170.041839:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041840:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879170.041843:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879170/real 1788879170] req@ffff94e50897b800 x1875765290136704/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879170.041848:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879170.041848:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879170.041850:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879170.041851:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879170.041851:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041852:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879170.041853:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879170.041856:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290136704/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879170.041861:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041862:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879170.041863:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041864:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879170.041865:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041866:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879170.041877:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879170.041878:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879170.041880:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879170.041881:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879170.041883:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290136704:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879170.041885:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879170.041886:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879170.041887:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879170.041888:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879170.041890:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879170.041891:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879170.041893:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290136832:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879170.041895:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879170.041896:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879170.041897:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879170.041899:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879170.041933:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041937:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879170.041938:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:2.0:1788879170.041940:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879170.041948:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879170.041951:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879170.041952:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879170.041953:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041955:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879170.041956:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041958:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304781322 00000100:00000001:2.0:1788879170.041960:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304781865 00000100:00000001:2.0:1788879170.041962:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304781322 00000100:00000001:2.0:1788879170.041963:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304781865 00000100:00000001:2.0:1788879170.041965:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879170.041966:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.041967:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879170.041970:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879170/real 1788879170] req@ffff94e50897bb80 x1875765290136832/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879170.041974:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879170.041975:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879170.041977:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879170.041978:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879170.041979:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041980:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879170.041981:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879170.041984:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290136832/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879170.041992:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041994:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879170.041996:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041996:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879170.041997:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.041998:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879170.042010:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879170.042012:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879170.042014:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879170.042015:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879170.042016:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290136832:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879170.042019:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.042020:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879170.042021:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879170.042023:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879170.042024:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879170.042025:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879170.042026:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879170.042028:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879170.042029:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879170.042031:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879170.042032:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.042033:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879170.042034:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879170.042035:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879170.042036:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879170.042037:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879170.042038:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879170.042039:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879170.042039:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879170.042042:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879170.042043:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.042043:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879170.042044:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879170.042045:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879170.042048:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879170.042049:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879170.042049:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879170.042050:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879170.042051:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879170.042052:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879170.042053:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.042054:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879170.042057:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879170.042057:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879170.042059:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879170.042060:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879170.104955:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879170.104960:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879170.104961:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879170.104966:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879170.104968:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879170.104971:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879170.104973:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879170.104975:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879170.104977:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879170.808990:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879170.808995:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879170.809001:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879170.809003:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879170.809039:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879170.809042:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879170.809046:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879171.129003:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879171.129008:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879171.129010:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879171.129014:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879171.129016:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879171.129019:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879171.129021:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879171.129023:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879171.129025:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879171.832980:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879171.832990:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879171.832997:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879171.832998:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879171.833040:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879171.833043:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879171.833045:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879172.152969:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879172.152974:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879172.152976:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879172.152980:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879172.152983:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879172.152986:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879172.152988:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879172.152989:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879172.152992:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879172.856982:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879172.856987:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879172.856990:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879172.856991:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879172.857015:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879172.857017:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879172.857020:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879173.177000:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879173.177008:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879173.177011:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879173.177016:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879173.177021:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879173.177025:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879173.177028:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879173.177031:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879173.177034:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879173.881013:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879173.881020:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879173.881025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879173.881028:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879173.881031:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879173.881035:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879173.881038:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879173.881040:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879173.881042:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879173.881045:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879173.881092:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879173.881095:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879173.881098:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879174.201024:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879174.201034:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879174.201037:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879174.201043:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879174.201047:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879174.201051:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879174.201054:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879174.201057:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879174.201060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879174.904988:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879174.904993:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879174.904997:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879174.904999:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879174.905043:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879174.905047:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879174.905050:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879175.160972:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879175.160981:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879175.160984:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879175.160987:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879175.160990:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879175.160996:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10114 00000100:00080000:0.0:1788879175.161006:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879175.161009:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879175.161011:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879175.161031:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879175.161034:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879175.161037:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879175.161041:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879175.161043:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879175.161045:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879175.161050:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879175.161053:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879175.161056:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879175.161058:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879175.161060:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161062:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879175.161064:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879175.161073:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879175.161074:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161076:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879175.161078:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879175.161082:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161091:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879175.161098:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290136960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879175.161111:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879175.161114:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161128:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879175.161136:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879175.161139:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879175.161140:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879175.161141:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879175.161143:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879175.161145:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10114 00000100:00080000:0.0:1788879175.161148:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879175.161149:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879175.161150:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879175.161152:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879175.161153:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879175.161155:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879175.161170:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879175.161172:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879175.161173:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879175.161176:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879175.161178:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879175.161179:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879175.161183:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879175.161185:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161186:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879175.161190:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879175.161192:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161194:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879175.161198:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879175.161198:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161200:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879175.161201:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879175.161203:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161207:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879175.161210:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290137088/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879175.161218:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879175.161219:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879175.161221:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161232:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.161233:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788879175.161240:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879175.161240:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879175.161246:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788879175.161247:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879175.161249:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161251:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879175.161258:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879175.161261:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879175.161263:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10114 00000100:00080000:0.0:1788879175.161265:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879175.161266:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879175.161267:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879175.161267:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879175.161273:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788879175.161275:0:178134:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1788879175.161277:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879175.161281:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879175.161287:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00100000:2.0:1788879175.161288:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290136960:192.168.203.129@tcp:250:kworker.0 00000020:00000001:0.0:1788879175.161298:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879175.161298:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879175.161305:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788879175.161306:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1788879175.161309:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879175.161313:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879175.161319:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879175.161322:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879175.161327:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879175.161329:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879175.161331:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879175.161335:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879175.161336:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161338:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879175.161341:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879175.161343:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.161344:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879175.161345:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.161348:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 02000000:00000001:0.0:1788879175.161353:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879175.161354:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1788879175.161356:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161359:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879175.161360:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879175.161362:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161367:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879175.161372:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290137216/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879175.161378:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879175.161380:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161381:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879175.161384:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879175.161387:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879175.161389:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879175.161391:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879175.161393:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879175.161395:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10114 00000100:00080000:0.0:1788879175.161399:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879175.161400:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879175.161402:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:2.0:1788879175.161406:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:0.0:1788879175.161409:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879175.161412:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879175.161414:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879175.161418:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879175.161431:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879175.161432:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879175.161436:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879175.161438:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879175.161440:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879175.161444:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879175.161444:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:0.0:1788879175.161449:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161452:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879175.161454:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879175.161454:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879175.161461:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788879175.161463:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161482:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.161482:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:0.0:1788879175.161497:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879175.161498:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879175.161504:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.161506:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304786985 00000100:00000001:0.0:1788879175.161511:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879175.161515:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879175.161518:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.161521:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788879175.161527:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.161535:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304786985 00000100:00100000:0.0:1788879175.161541:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290137344/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879175.161544:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879175.161554:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879175.161559:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879175.161562:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879175.161563:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.161567:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879175.161570:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879175.161595:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879175/real 1788879175] req@ffff94e508978e00 x1875765290136960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879175.161609:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879175.161611:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879175.161616:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879175.161618:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879175.161620:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.161622:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879175.161624:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879175.161629:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290136960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879175.161636:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.161640:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879175.161643:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.161644:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879175.161646:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.161648:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879175.161650:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879175.161653:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879175.161657:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879175.161659:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879175.161662:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290136960:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879175.161667:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.161670:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879175.161671:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879175.161674:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879175.161678:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879175.161698:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879175.161701:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879175.161706:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879175.161708:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879175.161712:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879175.161713:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.161715:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.161740:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879175.161741:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.161744:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879175.161745:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879175.161747:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879175.161748:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879175.161749:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879175.161752:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879175.161753:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879175.161755:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879175.161758:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290137088:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879175.161761:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879175.161763:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879175.161764:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879175.161766:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879175.161768:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.161774:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879175.161775:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:2.0:1788879175.161780:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879175.161808:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879175.161812:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879175.161814:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879175.161816:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.161825:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879175.161828:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.161833:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304786443 00000100:00000001:2.0:1788879175.161836:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304786985 00000100:00000001:2.0:1788879175.161838:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304786443 00000100:00000001:2.0:1788879175.161841:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304786985 00000100:00000001:2.0:1788879175.161843:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879175.161845:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.161846:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879175.161850:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879175/real 1788879175] req@ffff94e50897ad80 x1875765290137088/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879175.161858:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879175.161859:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879175.161862:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879175.161863:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879175.161864:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.161866:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879175.161867:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879175.161870:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290137088/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879175.161876:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.161878:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879175.161880:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.161881:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879175.161882:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.161884:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879175.161941:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879175.161944:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879175.161947:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879175.161949:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879175.161951:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290137088:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879175.161954:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879175.161956:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879175.161957:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879175.161959:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879175.161961:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879175.161963:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879175.161966:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290137216:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879175.161969:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879175.161971:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879175.161973:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879175.161974:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879175.161977:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.162008:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879175.162010:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:2.0:1788879175.162015:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879175.162029:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879175.162034:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879175.162036:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879175.162038:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.162039:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879175.162042:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.162046:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304786442 00000100:00000001:2.0:1788879175.162048:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304786985 00000100:00000001:2.0:1788879175.162051:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304786442 00000100:00000001:2.0:1788879175.162053:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304786985 00000100:00000001:2.0:1788879175.162055:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879175.162057:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.162059:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879175.162063:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879175/real 1788879175] req@ffff94e508979c00 x1875765290137216/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879175.162070:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879175.162072:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879175.162074:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879175.162076:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879175.162077:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.162079:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879175.162081:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879175.162085:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290137216/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879175.162098:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.162101:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879175.162103:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.162105:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879175.162106:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.162108:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879175.162124:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879175.162126:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879175.162129:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879175.162131:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879175.162134:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290137216:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879175.162138:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879175.162140:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879175.162141:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879175.162143:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879175.162161:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879175.162163:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879175.162165:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290137344:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879175.162167:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879175.162169:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879175.162171:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879175.162173:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879175.162176:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.162182:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879175.162184:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:2.0:1788879175.162188:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879175.162202:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879175.162207:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879175.162210:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879175.162212:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.162214:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879175.162217:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.162222:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304786442 00000100:00000001:2.0:1788879175.162225:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304786985 00000100:00000001:2.0:1788879175.162228:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304786442 00000100:00000001:2.0:1788879175.162231:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304786985 00000100:00000001:2.0:1788879175.162234:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879175.162236:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.162238:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879175.162244:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879175/real 1788879175] req@ffff94e50897a300 x1875765290137344/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879175.162254:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879175.162255:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879175.162257:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879175.162259:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879175.162260:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.162261:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879175.162262:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879175.162265:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290137344/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879175.162271:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.162273:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879175.162274:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.162276:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879175.162276:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.162278:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879175.162291:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879175.162292:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879175.162294:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879175.162296:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879175.162299:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290137344:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879175.162302:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.162303:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879175.162304:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879175.162306:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879175.162307:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879175.162308:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879175.162309:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879175.162312:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879175.162313:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879175.162317:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879175.162318:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.162319:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879175.162320:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879175.162321:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879175.162322:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879175.162323:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879175.162324:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879175.162326:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879175.162326:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879175.162328:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879175.162328:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.162329:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879175.162330:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879175.162331:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879175.162332:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879175.162333:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879175.162333:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879175.162334:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879175.162335:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879175.162336:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879175.162337:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.162338:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879175.162341:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879175.162342:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879175.162343:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879175.162344:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879175.224986:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879175.224991:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879175.224993:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879175.224997:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879175.225003:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879175.225006:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879175.225008:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879175.225011:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879175.225014:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879175.928981:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879175.928987:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879175.928989:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879175.928992:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879175.928995:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879175.928997:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879175.928998:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879175.929000:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879175.929002:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879175.929003:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879175.929018:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879175.929020:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879175.929022:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879176.249020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879176.249028:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879176.249030:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879176.249036:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879176.249040:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879176.249045:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879176.249048:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879176.249051:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879176.249054:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879176.953016:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879176.953021:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879176.953026:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879176.953027:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879176.953059:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879176.953062:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879176.953065:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879177.272959:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879177.272964:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879177.272966:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879177.272970:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879177.272973:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879177.272976:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879177.272978:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879177.272980:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879177.272982:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879177.977050:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879177.977059:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879177.977066:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879177.977068:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879177.977110:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879177.977114:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879177.977120:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879178.297054:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879178.297065:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879178.297069:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879178.297077:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879178.297084:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879178.297091:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879178.297095:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879178.297099:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879178.297103:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879179.001042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879179.001048:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879179.001054:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879179.001055:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879179.001111:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879179.001116:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879179.001120:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879179.321008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879179.321016:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879179.321017:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879179.321023:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879179.321026:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879179.321030:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879179.321032:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879179.321035:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879179.321037:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879180.025002:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879180.025007:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879180.025011:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879180.025012:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879180.025028:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879180.025030:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879180.025033:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879180.281020:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879180.281028:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879180.281030:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879180.281033:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879180.281055:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879180.281062:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10119 00000100:00080000:1.0:1788879180.281073:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879180.281077:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879180.281079:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879180.281088:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879180.281091:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879180.281094:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879180.281097:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879180.281099:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879180.281101:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879180.281106:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879180.281108:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879180.281111:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879180.281115:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879180.281119:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.281122:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879180.281124:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879180.281137:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879180.281139:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.281143:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879180.281146:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879180.281151:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.281164:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879180.281174:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100a680 x1875765290137472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879180.281202:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879180.281207:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.281227:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879180.281231:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879180.281235:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879180.281237:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879180.281239:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879180.281242:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879180.281245:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10119 00000100:00080000:1.0:1788879180.281250:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879180.281252:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879180.281254:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879180.281258:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879180.281259:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879180.281278:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879180.281280:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879180.281282:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879180.281283:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879180.281285:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879180.281286:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879180.281288:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879180.281293:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879180.281295:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.281296:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879180.281300:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879180.281302:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.281304:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879180.281308:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879180.281309:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.281311:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879180.281312:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879180.281313:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.281318:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.281319:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1788879180.281324:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009f80 x1875765290137600/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879180.281324:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879180.281345:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879180.281350:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879180.281355:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.281357:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788879180.281363:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788879180.281364:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879180.281370:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879180.281372:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879180.281377:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879180.281379:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879180.281387:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1788879180.281388:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290137472:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879180.281390:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879180.281395:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879180.281399:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788879180.281401:0:178134:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:2.0:1788879180.281406:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1788879180.281409:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10119 02000000:00000001:2.0:1788879180.281411:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879180.281419:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1788879180.281419:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788879180.281425:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879180.281427:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879180.281428:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879180.281432:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879180.281435:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879180.281438:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879180.281440:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879180.281443:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879180.281446:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879180.281449:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879180.281450:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879180.281454:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.281455:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286517888 : -117763423033728 : ffff94e51100a680) 02000000:00000001:1.0:1788879180.281460:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879180.281464:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1788879180.281466:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879180.281471:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.281473:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879180.281475:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879180.281478:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.281480:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879180.281503:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879180.281504:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.281505:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879180.281506:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879180.281507:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.281515:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879180.281519:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009180 x1875765290137728/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788879180.281526:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879180.281532:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879180.281534:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.281537:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879180.281539:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879180.281541:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:1.0:1788879180.281543:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879180.281546:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879180.281547:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1788879180.281548:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879180.281569:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.281570:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879180.281574:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:1.0:1788879180.281579:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10119 00000100:00000001:2.0:1788879180.281585:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879180.281587:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879180.281592:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879180.281593:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304792105 00000100:00000001:1.0:1788879180.281594:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879180.281603:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879180.281605:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879180.281608:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879180.281609:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:1.0:1788879180.281615:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879180.281620:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304792105 00000100:00000001:1.0:1788879180.281623:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879180.281627:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879180.281629:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879180.281632:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879180.281641:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879180.281663:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879180.281672:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.281674:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1788879180.281681:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879180.281688:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879180.281710:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879180.281711:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879180/real 1788879180] req@ffff94e51100a680 x1875765290137472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879180.281716:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879180.281721:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879180.281726:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879180.281726:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1788879180.281729:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879180.281730:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1788879180.281737:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879180.281737:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879180.281742:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879180.281743:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.281745:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879180.281747:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788879180.281749:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879180.281752:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100a680 x1875765290137472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879180.281753:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879180.281756:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879180.281759:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.281767:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.281767:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.281771:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:1.0:1788879180.281776:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008380 x1875765290137856/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879180.281776:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.281781:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879180.281785:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879180.281789:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879180.281791:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879180.281792:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.281797:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879180.281798:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879180.281805:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879180.281812:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879180.281815:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879180.281821:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290137472:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879180.281828:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.281830:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879180.281832:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879180.281835:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879180.281838:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879180.281839:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879180.281841:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879180.281845:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879180.281846:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879180.281848:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879180.281850:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.281851:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.281857:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879180.281859:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.281860:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879180.281861:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879180.281863:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879180.281864:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879180.281865:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879180.281867:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879180.281869:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879180.281871:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879180.281873:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290137600:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879180.281876:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879180.281878:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879180.281879:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879180.281880:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879180.281883:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.281888:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879180.281889:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286516096 : -117763423035520 : ffff94e511009f80) 00000100:00000001:2.0:1788879180.281893:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879180.281903:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879180.281907:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879180.281908:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879180.281909:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.281911:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879180.281913:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.281916:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304791563 00000100:00000001:2.0:1788879180.281943:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304792105 00000100:00000001:2.0:1788879180.281945:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304791564 00000100:00000001:2.0:1788879180.281947:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304792105 00000100:00000001:2.0:1788879180.281948:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879180.281950:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.281951:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879180.281954:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879180/real 1788879180] req@ffff94e511009f80 x1875765290137600/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879180.281960:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879180.281961:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879180.281963:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879180.281965:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879180.281967:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.281968:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879180.281970:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879180.281974:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009f80 x1875765290137600/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879180.281983:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.281986:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879180.281988:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.281990:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879180.281992:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.281994:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879180.282008:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879180.282012:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879180.282016:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879180.282018:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879180.282022:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290137600:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879180.282026:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879180.282027:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879180.282029:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879180.282031:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879180.282034:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879180.282036:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879180.282039:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290137728:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879180.282043:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879180.282045:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879180.282047:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879180.282049:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879180.282052:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.282058:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879180.282060:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286512512 : -117763423039104 : ffff94e511009180) 00000100:00000001:2.0:1788879180.282064:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879180.282079:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879180.282084:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879180.282087:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879180.282089:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.282091:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879180.282094:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.282097:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304791562 00000100:00000001:2.0:1788879180.282100:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304792105 00000100:00000001:2.0:1788879180.282102:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304791562 00000100:00000001:2.0:1788879180.282105:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304792105 00000100:00000001:2.0:1788879180.282107:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879180.282109:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.282111:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879180.282116:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879180/real 1788879180] req@ffff94e511009180 x1875765290137728/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879180.282124:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879180.282126:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879180.282129:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879180.282130:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879180.282132:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.282134:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879180.282136:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879180.282141:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009180 x1875765290137728/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879180.282148:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.282151:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879180.282153:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.282155:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879180.282156:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.282158:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879180.282172:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879180.282175:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879180.282179:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879180.282181:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879180.282185:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290137728:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879180.282189:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879180.282190:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879180.282192:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879180.282194:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879180.282197:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879180.282199:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879180.282203:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290137856:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879180.282206:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879180.282208:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879180.282210:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879180.282212:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879180.282215:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.282221:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879180.282223:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508928 : -117763423042688 : ffff94e511008380) 00000100:00000001:2.0:1788879180.282227:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879180.282241:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879180.282266:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879180.282270:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879180.282273:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.282275:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879180.282280:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.282285:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304791562 00000100:00000001:2.0:1788879180.282289:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304792105 00000100:00000001:2.0:1788879180.282292:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304791562 00000100:00000001:2.0:1788879180.282296:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304792105 00000100:00000001:2.0:1788879180.282299:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879180.282303:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.282306:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879180.282313:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879180/real 1788879180] req@ffff94e511008380 x1875765290137856/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879180.282356:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879180.282358:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879180.282363:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879180.282365:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879180.282368:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.282371:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879180.282373:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879180.282380:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008380 x1875765290137856/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879180.282392:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.282396:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879180.282401:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.282403:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879180.282406:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.282409:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879180.282429:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879180.282435:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879180.282440:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879180.282444:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879180.282450:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290137856:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879180.282457:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.282460:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879180.282463:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879180.282465:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879180.282470:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879180.282473:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879180.282475:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879180.282478:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879180.282497:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879180.282499:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879180.282501:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.282503:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879180.282505:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879180.282507:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879180.282509:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879180.282510:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879180.282511:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879180.282514:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879180.282515:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879180.282517:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879180.282518:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.282520:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879180.282521:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879180.282522:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879180.282524:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879180.282525:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879180.282527:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879180.282529:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879180.282530:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879180.282531:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879180.282533:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.282535:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879180.282539:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879180.282541:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879180.282543:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879180.282545:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879180.344988:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879180.344995:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879180.344997:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879180.345002:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879180.345006:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879180.345009:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879180.345012:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879180.345014:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879180.345017:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879181.048972:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879181.048979:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879181.048985:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879181.048987:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879181.049008:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879181.049011:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879181.049014:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879181.368982:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879181.368987:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879181.368989:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879181.368992:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879181.368995:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879181.368999:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879181.369001:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879181.369003:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879181.369005:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879182.073075:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879182.073085:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879182.073092:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879182.073095:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879182.073173:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879182.073203:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879182.073208:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879182.393001:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879182.393010:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879182.393014:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879182.393021:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879182.393025:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879182.393030:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879182.393034:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879182.393038:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879182.393042:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879183.097067:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879183.097077:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879183.097084:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879183.097089:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879183.097093:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879183.097098:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879183.097102:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879183.097105:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879183.097108:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879183.097111:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879183.097260:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879183.097267:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879183.097273:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879183.416994:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879183.417000:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879183.417002:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879183.417006:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879183.417009:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879183.417012:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879183.417014:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879183.417016:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879183.417018:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879184.121046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879184.121054:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879184.121060:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879184.121062:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879184.121140:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879184.121143:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879184.121147:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879184.441055:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879184.441081:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879184.441084:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879184.441090:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879184.441094:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879184.441098:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879184.441102:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879184.441105:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879184.441108:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879185.145034:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879185.145044:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879185.145046:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879185.145049:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879185.145054:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879185.145056:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879185.145057:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879185.145059:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879185.145063:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879185.145064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879185.145089:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879185.145091:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879185.145093:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879185.401069:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879185.401079:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879185.401082:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879185.401086:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879185.401090:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879185.401098:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10124 00000100:00080000:0.0:1788879185.401110:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879185.401113:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879185.401116:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879185.401124:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879185.401127:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879185.401130:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879185.401132:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879185.401134:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879185.401135:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879185.401150:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879185.401151:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879185.401154:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879185.401158:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879185.401160:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401162:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879185.401164:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879185.401179:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879185.401180:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401184:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879185.401186:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879185.401192:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401206:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879185.401216:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290137984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879185.401226:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879185.401231:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401256:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879185.401259:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879185.401262:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879185.401263:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879185.401264:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879185.401267:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879185.401269:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10124 00000100:00080000:0.0:1788879185.401271:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879185.401273:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879185.401274:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879185.401276:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879185.401277:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879185.401279:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879185.401280:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879185.401282:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879185.401283:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879185.401284:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879185.401286:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879185.401287:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879185.401292:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879185.401294:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401296:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879185.401300:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879185.401303:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401304:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879185.401308:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879185.401309:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401310:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879185.401311:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879185.401313:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.401314:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879185.401318:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.401320:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879185.401324:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00100000:0.0:1788879185.401325:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897aa00 x1875765290138112/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879185.401327:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879185.401333:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879185.401335:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401337:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879185.401339:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879185.401344:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879185.401345:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879185.401349:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879185.401350:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1788879185.401351:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788879185.401354:0:136860:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:2.0:1788879185.401356:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1788879185.401359:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10124 02000000:00000001:2.0:1788879185.401361:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879185.401363:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879185.401368:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879185.401370:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879185.401372:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879185.401373:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:2.0:1788879185.401373:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290137984:192.168.203.129@tcp:250:kworker.0 00000020:00000001:0.0:1788879185.401377:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879185.401382:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879185.401383:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788879185.401384:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879185.401404:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879185.401407:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1788879185.401408:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:2.0:1788879185.401411:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401412:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879185.401417:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879185.401417:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879185.401423:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879185.401426:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879185.401426:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401428:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879185.401433:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879185.401436:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401437:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879185.401442:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879185.401443:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401444:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879185.401445:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879185.401447:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.401451:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879185.401453:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.401456:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00100000:0.0:1788879185.401461:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978700 x1875765290138240/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879185.401462:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879185.401471:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879185.401474:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401476:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879185.401478:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879185.401482:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879185.401483:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879185.401485:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879185.401488:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879185.401490:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10124 00000100:00080000:0.0:1788879185.401493:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879185.401495:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879185.401496:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879185.401502:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879185.401503:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879185.401505:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879185.401506:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879185.401508:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879185.401509:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879185.401512:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879185.401517:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788879185.401519:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:0.0:1788879185.401520:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879185.401525:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879185.401527:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401529:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879185.401530:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879185.401532:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401534:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879185.401539:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879185.401540:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401541:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879185.401551:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879185.401552:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.401555:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879185.401557:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879185.401565:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290138368/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879185.401565:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879185.401572:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.401578:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879185.401604:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879185.401608:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879185.401629:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.401631:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304797225 00000100:00000001:0.0:1788879185.401633:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.401645:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788879185.401654:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304797225 00000100:00000001:2.0:1788879185.401669:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879185.401672:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879185.401675:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.401678:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879185.401689:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879185/real 1788879185] req@ffff94e508978a80 x1875765290137984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879185.401701:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879185.401703:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879185.401708:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879185.401710:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879185.401712:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.401714:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879185.401716:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879185.401720:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290137984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879185.401726:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.401729:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879185.401731:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.401732:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879185.401733:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.401735:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879185.401736:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879185.401739:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879185.401743:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879185.401745:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879185.401748:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290137984:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879185.401752:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.401754:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879185.401755:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879185.401758:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879185.401760:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879185.401762:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879185.401764:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879185.401767:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879185.401769:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879185.401771:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879185.401772:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.401774:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.401781:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879185.401783:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.401784:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879185.401786:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879185.401787:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879185.401788:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879185.401789:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879185.401791:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879185.401792:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879185.401794:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879185.401796:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290138112:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879185.401799:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879185.401800:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879185.401801:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879185.401803:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879185.401805:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.401811:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879185.401813:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:2.0:1788879185.401819:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879185.401834:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879185.401839:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879185.401842:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879185.401844:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.401846:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879185.401849:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.401854:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304796683 00000100:00000001:2.0:1788879185.401858:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304797225 00000100:00000001:2.0:1788879185.401861:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304796683 00000100:00000001:2.0:1788879185.401864:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304797225 00000100:00000001:2.0:1788879185.401866:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879185.401869:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.401871:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879185.401875:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879185/real 1788879185] req@ffff94e50897aa00 x1875765290138112/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879185.401882:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879185.401883:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879185.401885:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879185.401887:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879185.401888:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.401889:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879185.401890:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879185.401893:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290138112/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879185.401898:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.401899:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879185.401934:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.401937:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879185.401938:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.401940:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879185.401959:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879185.401962:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879185.401968:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879185.401970:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879185.401974:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290138112:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879185.401980:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879185.401982:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879185.401983:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879185.401986:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879185.401988:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879185.401990:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879185.401993:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290138240:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879185.401995:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879185.401997:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879185.401998:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879185.402000:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879185.402002:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.402008:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879185.402010:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00000001:2.0:1788879185.402013:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879185.402024:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879185.402027:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879185.402029:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879185.402030:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.402032:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879185.402034:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.402037:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304796682 00000100:00000001:2.0:1788879185.402039:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304797225 00000100:00000001:2.0:1788879185.402040:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304796682 00000100:00000001:2.0:1788879185.402042:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304797225 00000100:00000001:2.0:1788879185.402044:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879185.402045:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.402047:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879185.402050:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879185/real 1788879185] req@ffff94e508978700 x1875765290138240/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879185.402056:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879185.402057:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879185.402059:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879185.402060:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879185.402062:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.402063:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879185.402064:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879185.402067:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290138240/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879185.402071:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.402073:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879185.402075:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.402076:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879185.402077:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.402078:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879185.402090:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879185.402092:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879185.402094:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879185.402095:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879185.402097:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290138240:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879185.402100:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879185.402102:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879185.402103:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879185.402104:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879185.402105:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879185.402107:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879185.402108:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290138368:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879185.402110:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879185.402112:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879185.402113:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879185.402114:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879185.402117:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.402119:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879185.402121:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:2.0:1788879185.402123:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879185.402129:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879185.402132:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879185.402144:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879185.402145:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.402146:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879185.402148:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.402150:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304796682 00000100:00000001:2.0:1788879185.402152:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304797225 00000100:00000001:2.0:1788879185.402154:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304796682 00000100:00000001:2.0:1788879185.402156:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304797225 00000100:00000001:2.0:1788879185.402158:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879185.402159:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.402160:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879185.402162:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879185/real 1788879185] req@ffff94e50897a300 x1875765290138368/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879185.402167:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879185.402168:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879185.402169:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879185.402170:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879185.402171:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.402172:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879185.402173:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879185.402175:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290138368/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879185.402180:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.402182:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879185.402184:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.402185:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879185.402186:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.402187:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879185.402202:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879185.402204:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879185.402207:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879185.402208:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879185.402211:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290138368:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879185.402213:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.402214:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879185.402215:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879185.402216:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879185.402217:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879185.402218:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879185.402219:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879185.402220:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879185.402221:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879185.402224:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879185.402225:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.402226:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879185.402227:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879185.402228:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879185.402230:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879185.402231:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879185.402231:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879185.402233:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879185.402234:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879185.402234:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879185.402235:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.402236:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879185.402237:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879185.402238:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879185.402239:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879185.402240:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879185.402240:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879185.402241:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879185.402243:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879185.402244:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879185.402245:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.402246:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879185.402249:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879185.402250:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879185.402252:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879185.402253:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879185.464999:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879185.465005:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879185.465007:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879185.465010:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879185.465013:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879185.465016:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879185.465018:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879185.465019:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879185.465021:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879186.169049:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879186.169056:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879186.169062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879186.169063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879186.169086:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879186.169090:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879186.169093:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879186.489052:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879186.489062:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879186.489065:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879186.489073:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879186.489078:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879186.489083:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879186.489086:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879186.489101:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879186.489105:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879187.193119:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879187.193129:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879187.193136:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879187.193138:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879187.193308:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879187.193315:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879187.193320:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879187.513052:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879187.513062:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879187.513064:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879187.513070:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879187.513074:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879187.513078:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879187.513081:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879187.513084:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879187.513087:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879188.217079:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879188.217092:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879188.217102:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879188.217104:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879188.217137:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879188.217140:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879188.217145:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879188.537040:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879188.537048:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879188.537050:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879188.537056:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879188.537061:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879188.537065:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879188.537069:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879188.537071:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879188.537074:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879189.241075:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879189.241083:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879189.241090:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879189.241091:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879189.241157:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879189.241161:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879189.241165:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879189.561034:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879189.561046:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879189.561051:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879189.561061:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879189.561069:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879189.561076:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879189.561081:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879189.561086:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879189.561091:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879190.265001:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879190.265008:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879190.265014:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879190.265015:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879190.265051:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879190.265054:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879190.265057:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879190.521140:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879190.521158:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879190.521163:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879190.521177:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879190.521186:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879190.521206:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10129 00000100:00080000:1.0:1788879190.521226:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879190.521233:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879190.521238:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879190.521250:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879190.521255:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879190.521260:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879190.521265:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879190.521269:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879190.521272:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879190.521282:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879190.521287:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879190.521293:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879190.521300:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879190.521306:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879190.521310:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879190.521326:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879190.521343:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879190.521347:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879190.521353:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879190.521357:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879190.521364:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879190.521383:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879190.521402:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009880 x1875765290138496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879190.521423:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879190.521431:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879190.521464:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879190.521472:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879190.521479:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879190.521482:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879190.521486:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879190.521492:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879190.521497:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10129 00000100:00080000:1.0:1788879190.521506:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879190.521510:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879190.521512:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879190.521517:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879190.521519:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879190.521522:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879190.521525:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879190.521527:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879190.521529:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879190.521533:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879190.521537:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879190.521540:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879190.521549:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879190.521552:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879190.521556:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879190.521562:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879190.521566:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879190.521569:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.521596:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879190.521611:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1788879190.521612:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879190.521617:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1788879190.521618:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.521628:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879190.521630:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879190.521634:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879190.521655:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879190.521656:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879190.521661:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879190.521664:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879190.521674:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879190.521678:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879190.521688:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290138496:192.168.203.129@tcp:250:kworker.0 00000100:00100000:1.0:1788879190.521695:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008380 x1875765290138624/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879190.521704:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879190.521715:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1788879190.521716:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788879190.521728:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879190.521731:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879190.521735:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788879190.521742:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879190.521746:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879190.521749:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879190.521756:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879190.521758:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879190.521760:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879190.521765:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879190.521770:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10129 00000100:00080000:1.0:1788879190.521777:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879190.521777:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879190.521783:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286514304 : -117763423037312 : ffff94e511009880) 00000100:00000001:1.0:1788879190.521785:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879190.521795:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879190.521798:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1788879190.521814:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879190.521818:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879190.521823:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879190.521828:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879190.521832:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879190.521835:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879190.521841:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879190.521847:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879190.521852:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879190.521859:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879190.521865:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788879190.521874:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879190.521876:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879190.521885:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879190.521896:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.521897:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879190.521905:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.521957:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879190.521962:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.521965:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879190.521973:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.521980:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304802346 02000000:00000001:1.0:1788879190.521981:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879190.521989:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879190.521995:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879190.521998:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879190.522000:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1788879190.522006:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.522013:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304802346 00000100:00000001:2.0:1788879190.522021:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879190.522026:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.522028:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879190.522037:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.522041:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879190.522049:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009180 x1875765290138752/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788879190.522056:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879190/real 1788879190] req@ffff94e511009880 x1875765290138496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879190.522118:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879190.522124:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879190.522128:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1788879190.522132:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879190.522134:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:1.0:1788879190.522141:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879190.522145:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:1.0:1788879190.522148:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879190.522156:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879190.522156:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879190.522167:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788879190.522169:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879190.522188:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879190.522205:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:1.0:1788879190.522207:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10129 00000100:00000001:2.0:1788879190.522211:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788879190.522218:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879190.522227:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1788879190.522227:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009880 x1875765290138496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879190.522240:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879190.522258:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879190.522265:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1788879190.522266:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879190.522279:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879190.522282:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:1.0:1788879190.522289:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879190.522293:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879190.522298:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879190.522300:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879190.522305:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.522321:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879190.522325:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879190.522327:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879190.522345:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879190.522348:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879190.522354:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879190.522355:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 02000000:00000001:1.0:1788879190.522370:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879190.522371:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:1.0:1788879190.522382:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1788879190.522383:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290138496:192.168.203.129@tcp:250:kworker.0 02000000:00000001:1.0:1788879190.522407:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.522407:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.522421:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879190.522427:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879190.522427:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879190.522433:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000008:00100000:1.0:1788879190.522438:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788879190.522440:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879190.522445:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879190.522451:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000008:00000001:1.0:1788879190.522454:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879190.522462:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1788879190.522464:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879190.522467:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879190.522474:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879190.522477:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.522480:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788879190.522482:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879190.522488:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.522491:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879190.522495:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879190.522496:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879190.522500:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879190.522504:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1788879190.522508:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.522526:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879190.522531:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879190.522533:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.522534:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879190.522542:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879190.522548:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:1.0:1788879190.522552:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009f80 x1875765290138880/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788879190.522552:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879190.522571:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879190.522641:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290138624:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879190.522648:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879190.522650:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879190.522656:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1788879190.522660:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879190.522660:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879190.522664:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879190.522667:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.522676:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879190.522680:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508928 : -117763423042688 : ffff94e511008380) 00000100:00000001:1.0:1788879190.522686:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.522687:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879190.522714:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879190.522721:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879190.522725:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879190.522726:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.522729:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879190.522733:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.522739:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304801804 00000100:00000001:2.0:1788879190.522743:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304802346 00000100:00000001:2.0:1788879190.522747:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304801804 00000100:00000001:2.0:1788879190.522749:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304802346 00000100:00000001:2.0:1788879190.522752:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879190.522754:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.522756:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879190.522762:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879190/real 1788879190] req@ffff94e511008380 x1875765290138624/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879190.522772:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879190.522773:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879190.522776:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879190.522778:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879190.522779:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.522781:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879190.522783:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879190.522786:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008380 x1875765290138624/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879190.522794:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.522796:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879190.522798:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.522800:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879190.522801:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.522803:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879190.522823:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879190.522825:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879190.522828:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879190.522831:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879190.522834:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290138624:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879190.522837:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879190.522839:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879190.522841:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879190.522844:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879190.522846:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879190.522848:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879190.522851:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290138752:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879190.522857:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879190.522859:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879190.522877:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879190.522880:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879190.522884:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.522892:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879190.522894:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286512512 : -117763423039104 : ffff94e511009180) 00000100:00000001:2.0:1788879190.522910:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879190.522925:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879190.522930:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879190.522963:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879190.522965:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.522968:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879190.522972:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.522977:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304801803 00000100:00000001:2.0:1788879190.522981:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304802346 00000100:00000001:2.0:1788879190.522984:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304801803 00000100:00000001:2.0:1788879190.522987:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304802346 00000100:00000001:2.0:1788879190.522990:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879190.522992:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.522994:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879190.523000:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879190/real 1788879190] req@ffff94e511009180 x1875765290138752/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879190.523009:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879190.523010:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879190.523017:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879190.523019:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879190.523022:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.523024:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879190.523026:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879190.523030:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009180 x1875765290138752/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879190.523048:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.523051:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879190.523054:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.523056:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879190.523057:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.523060:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879190.523078:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879190.523081:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879190.523085:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879190.523087:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879190.523091:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290138752:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879190.523096:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.523098:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879190.523100:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879190.523102:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879190.523116:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879190.523118:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879190.523120:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879190.523123:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879190.523125:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879190.523127:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879190.523129:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.523131:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879190.523133:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879190.523134:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879190.523137:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879190.523138:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879190.523140:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879190.523142:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879190.523144:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879190.523145:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879190.523147:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.523149:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.523153:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879190.523155:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.523158:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879190.523160:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879190.523162:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879190.523164:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879190.523166:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879190.523186:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879190.523187:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879190.523189:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879190.523193:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290138880:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879190.523197:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879190.523199:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879190.523201:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879190.523203:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879190.523206:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.523212:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879190.523213:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286516096 : -117763423035520 : ffff94e511009f80) 00000100:00000001:2.0:1788879190.523218:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879190.523231:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879190.523252:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879190.523255:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879190.523257:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.523259:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879190.523262:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.523279:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304801803 00000100:00000001:2.0:1788879190.523282:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304802346 00000100:00000001:2.0:1788879190.523285:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304801803 00000100:00000001:2.0:1788879190.523288:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304802346 00000100:00000001:2.0:1788879190.523291:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879190.523293:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.523296:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879190.523301:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879190/real 1788879190] req@ffff94e511009f80 x1875765290138880/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879190.523319:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879190.523321:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879190.523324:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879190.523326:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879190.523327:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.523329:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879190.523331:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879190.523335:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009f80 x1875765290138880/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879190.523343:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.523346:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879190.523348:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.523350:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879190.523358:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.523361:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879190.523379:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879190.523383:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879190.523387:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879190.523390:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879190.523394:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290138880:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879190.523398:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.523400:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879190.523402:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879190.523403:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879190.523406:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879190.523407:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879190.523409:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879190.523411:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879190.523413:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879190.523415:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879190.523416:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.523419:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879190.523423:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879190.523424:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879190.523427:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879190.523429:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879190.585021:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879190.585029:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879190.585031:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879190.585047:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879190.585052:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879190.585057:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879190.585061:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879190.585064:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879190.585067:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879191.289053:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879191.289061:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879191.289067:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879191.289069:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879191.289232:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879191.289240:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879191.289245:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879191.608986:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879191.608996:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879191.608999:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879191.609006:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879191.609011:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879191.609015:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879191.609019:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879191.609024:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879191.609029:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879192.313079:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879192.313089:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879192.313097:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879192.313099:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879192.313133:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879192.313137:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879192.313141:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879192.633091:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879192.633115:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879192.633119:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879192.633128:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879192.633134:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879192.633140:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879192.633143:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879192.633147:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879192.633151:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879193.337046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879193.337054:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879193.337060:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879193.337063:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879193.337066:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879193.337071:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879193.337075:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879193.337078:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879193.337080:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879193.337083:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879193.337127:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879193.337130:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879193.337133:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879193.657039:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879193.657047:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879193.657050:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879193.657056:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879193.657060:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879193.657065:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879193.657068:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879193.657071:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879193.657075:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879194.361085:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879194.361093:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879194.361098:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879194.361100:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879194.361124:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879194.361127:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879194.361130:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879194.681004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879194.681011:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879194.681013:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879194.681019:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879194.681023:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879194.681027:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879194.681029:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879194.681032:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879194.681035:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879195.385007:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879195.385033:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879195.385036:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879195.385039:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879195.385044:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879195.385047:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879195.385049:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879195.385052:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879195.385055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879195.385057:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879195.385154:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879195.385159:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879195.385163:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879195.641045:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879195.641055:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879195.641058:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879195.641062:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879195.641068:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879195.641076:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10134 00000100:00080000:3.0:1788879195.641088:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879195.641092:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879195.641094:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879195.641103:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879195.641106:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879195.641109:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879195.641112:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879195.641114:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879195.641116:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879195.641121:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879195.641123:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879195.641126:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879195.641131:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879195.641144:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879195.641147:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879195.641150:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879195.641159:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879195.641161:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879195.641164:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879195.641168:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879195.641172:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879195.641184:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879195.641194:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290139008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879195.641206:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879195.641210:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879195.641225:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879195.641229:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879195.641233:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879195.641236:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879195.641239:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879195.641243:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879195.641247:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10134 00000100:00080000:3.0:1788879195.641252:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879195.641255:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879195.641256:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879195.641258:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879195.641266:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788879195.641266:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879195.641272:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788879195.641272:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879195.641277:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879195.641279:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879195.641284:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879195.641285:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879195.641292:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879195.641293:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879195.641294:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879195.641300:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879195.641302:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788879195.641306:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879195.641309:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879195.641316:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1788879195.641317:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290139008:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788879195.641323:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879195.641324:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879195.641331:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879195.641333:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788879195.641334:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879195.641338:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788879195.641341:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879195.641342:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1788879195.641348:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879195.641350:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879195.641352:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.641372:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879195.641377:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:1.0:1788879195.641384:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879195.641435:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879195.641453:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879195.641459:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879195.641462:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879195.641465:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879195.641466:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879195.641470:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879195.641474:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879195.641477:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879195.641478:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879195.641483:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.641486:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304807465 00000100:00000001:3.0:1788879195.641491:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.641499:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00100000:3.0:1788879195.641501:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290139136/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879195.641511:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304807465 00000100:00000001:3.0:1788879195.641512:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879195.641519:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.641523:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879195.641525:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879195.641532:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879195.641534:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879195.641535:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879195.641557:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.641565:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879195.641576:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879195/real 1788879195] req@ffff94e50449c700 x1875765290139008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879195.641594:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879195.641596:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879195.641601:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879195.641604:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879195.641606:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.641608:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879195.641610:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879195.641616:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290139008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879195.641624:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.641628:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879195.641631:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.641634:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879195.641635:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.641637:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879195.641640:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879195.641644:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879195.641648:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879195.641650:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879195.641654:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290139008:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879195.641659:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.641662:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879195.641664:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879195.641666:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879195.641671:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879195.641673:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879195.641675:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879195.641679:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879195.641681:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879195.641684:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879195.641686:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.641688:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.641695:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879195.641697:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879195.641699:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879195.641700:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879195.641704:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879195.641707:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1788879195.641708:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879195.641709:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879195.641713:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879195.641719:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879195.641721:0:178134:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:1.0:1788879195.641722:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879195.641728:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879195.641733:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10134 00000100:00100000:1.0:1788879195.641735:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290139136:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879195.641741:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879195.641743:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879195.641745:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879195.641748:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879195.641754:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.641764:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879195.641767:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:1.0:1788879195.641773:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879195.641799:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1788879195.641806:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879195.641812:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879195.641815:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879195.641816:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879195.641825:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1788879195.641826:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879195.641830:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879195.641832:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879195.641834:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879195.641839:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:3.0:1788879195.641840:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879195.641845:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879195.641847:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879195.641848:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879195.641858:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304806923 00000100:00080000:3.0:1788879195.641859:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879195.641866:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304807465 00000100:00000001:3.0:1788879195.641867:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.641875:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304806923 02000000:00000001:3.0:1788879195.641875:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879195.641882:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304807465 02000000:00000001:3.0:1788879195.641883:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879195.641886:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879195.641891:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.641892:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.641900:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879195.641938:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879195.641945:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879195.641951:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879195/real 1788879195] req@ffff94e50449f480 x1875765290139136/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1788879195.641951:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879195.641958:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879195.641970:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000400:00000001:1.0:1788879195.641972:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1788879195.641974:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879195.641976:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879195.641981:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879195.641984:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:3.0:1788879195.641985:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879195.641989:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:3.0:1788879195.641990:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.641993:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.641997:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879195.641999:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879195.642000:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879195.642004:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290139136/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879195.642012:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290139264/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879195.642016:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.642021:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879195.642028:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.642033:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879195.642036:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879195.642038:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879195.642040:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879195.642044:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879195.642053:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879195.642056:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879195.642061:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879195.642062:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879195.642063:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879195.642069:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879195.642071:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788879195.642077:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879195.642080:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879195.642086:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:3.0:1788879195.642087:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10134 00000100:00100000:1.0:1788879195.642096:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290139136:192.168.203.129@tcp:38:kworker.0 00000100:00080000:3.0:1788879195.642096:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879195.642104:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879195.642106:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879195.642107:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879195.642110:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879195.642114:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1788879195.642115:0:178134:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1788879195.642117:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:3.0:1788879195.642120:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1788879195.642122:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879195.642124:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:1.0:1788879195.642125:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879195.642130:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000020:00000001:3.0:1788879195.642131:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788879195.642134:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1788879195.642136:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879195.642137:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879195.642140:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879195.642142:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879195.642147:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788879195.642151:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879195.642152:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879195.642160:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.642161:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1788879195.642163:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879195.642165:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879195.642168:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879195.642170:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879195.642173:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788879195.642176:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.642178:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879195.642180:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879195.642184:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1788879195.642185:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879195.642190:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788879195.642191:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879195.642199:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879195.642200:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879195.642204:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879195.642207:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879195.642208:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879195.642217:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879195.642218:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879195.642219:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879195.642225:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879195.642232:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290139264:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879195.642234:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.642243:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1788879195.642245:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290139392/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879195.642249:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879195.642255:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879195.642261:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879195.642263:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879195.642266:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879195.642267:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879195.642277:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.642282:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879195.642285:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:1.0:1788879195.642290:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879195.642308:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879195.642313:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879195.642316:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879195.642317:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.642319:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879195.642322:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.642325:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304806922 00000100:00000001:1.0:1788879195.642328:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304807465 00000100:00000001:1.0:1788879195.642330:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304806922 00000100:00000001:1.0:1788879195.642332:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304807465 00000100:00000001:1.0:1788879195.642334:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879195.642336:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.642338:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879195.642342:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879195/real 1788879195] req@ffff94e50449f800 x1875765290139264/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879195.642350:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879195.642351:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879195.642353:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879195.642354:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879195.642356:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.642358:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879195.642359:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879195.642362:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290139264/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879195.642369:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.642371:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879195.642373:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.642374:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879195.642375:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.642379:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879195.642396:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879195.642400:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879195.642406:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879195.642409:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879195.642415:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290139264:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879195.642420:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.642423:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879195.642425:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879195.642428:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879195.642431:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879195.642434:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879195.642436:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879195.642439:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879195.642441:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879195.642444:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879195.642447:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.642450:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.642455:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879195.642457:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.642459:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879195.642461:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879195.642462:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879195.642464:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879195.642465:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879195.642468:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879195.642470:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879195.642472:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879195.642475:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290139392:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879195.642479:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879195.642481:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879195.642482:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879195.642484:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879195.642487:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.642494:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879195.642496:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:1.0:1788879195.642500:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879195.642518:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879195.642525:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879195.642528:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879195.642530:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.642533:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879195.642537:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.642561:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304806922 00000100:00000001:1.0:1788879195.642565:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304807465 00000100:00000001:1.0:1788879195.642568:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304806922 00000100:00000001:1.0:1788879195.642572:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304807465 00000100:00000001:1.0:1788879195.642576:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879195.642579:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.642582:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879195.642589:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879195/real 1788879195] req@ffff94e50449d880 x1875765290139392/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879195.642600:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879195.642602:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879195.642605:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879195.642607:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879195.642608:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.642610:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879195.642612:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879195.642616:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290139392/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879195.642622:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.642625:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879195.642627:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.642628:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879195.642630:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.642631:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879195.642644:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879195.642647:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879195.642650:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879195.642652:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879195.642655:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290139392:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879195.642658:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.642660:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879195.642661:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879195.642663:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879195.642665:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879195.642666:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879195.642668:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879195.642670:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879195.642671:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879195.642673:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879195.642675:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.642676:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.642679:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879195.642681:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.642683:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879195.642684:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879195.704988:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879195.704993:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879195.704995:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879195.705000:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879195.705003:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879195.705006:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.705009:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879195.705011:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879195.705014:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879196.408986:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879196.409009:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879196.409014:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879196.409015:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879196.409035:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879196.409037:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879196.409040:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879196.728976:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879196.728983:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879196.728986:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879196.728992:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879196.728997:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879196.729000:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879196.729003:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879196.729005:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879196.729008:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879197.433049:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879197.433058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879197.433066:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879197.433068:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879197.433150:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879197.433154:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879197.433158:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879197.753022:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879197.753030:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879197.753033:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879197.753039:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879197.753044:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879197.753049:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879197.753053:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879197.753056:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879197.753060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879198.457000:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879198.457007:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879198.457012:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879198.457013:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879198.457034:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879198.457037:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879198.457040:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879198.777023:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879198.777032:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879198.777035:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879198.777042:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879198.777046:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879198.777051:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879198.777055:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879198.777058:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879198.777061:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879199.481031:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879199.481040:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879199.481046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879199.481048:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879199.481177:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1788879199.481185:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1788879199.481191:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879199.801030:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879199.801039:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879199.801042:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879199.801048:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879199.801053:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879199.801058:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879199.801062:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879199.801065:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879199.801068:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879200.505030:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879200.505036:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879200.505041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879200.505042:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879200.505063:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879200.505066:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879200.505069:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879200.761047:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879200.761060:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879200.761063:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879200.761068:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879200.761087:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879200.761097:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10139 00000100:00080000:3.0:1788879200.761111:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879200.761115:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879200.761118:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879200.761128:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879200.761131:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879200.761136:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879200.761139:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879200.761142:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879200.761144:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879200.761150:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879200.761153:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879200.761156:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879200.761161:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879200.761165:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879200.761168:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879200.761171:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879200.761185:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879200.761187:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879200.761192:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879200.761195:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879200.761201:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879200.761215:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879200.761227:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290139520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879200.761241:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879200.761247:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879200.761267:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879200.761272:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879200.761276:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879200.761278:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879200.761281:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879200.761285:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879200.761289:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10139 00000100:00080000:3.0:1788879200.761295:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879200.761298:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879200.761301:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879200.761306:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879200.761308:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879200.761311:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879200.761314:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879200.761317:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879200.761318:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879200.761323:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879200.761327:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879200.761327:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879200.761336:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879200.761338:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879200.761343:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879200.761347:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879200.761350:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879200.761354:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879200.761358:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879200.761361:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879200.761365:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879200.761366:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879200.761375:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788879200.761377:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879200.761388:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290139520:192.168.203.129@tcp:250:kworker.0 00000008:00000001:3.0:1788879200.761389:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879200.761398:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.761401:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879200.761407:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879200.761410:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879200.761412:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879200.761417:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879200.761418:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879200.761424:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879200.761426:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879200.761430:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879200.761436:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879200.761443:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879200.761456:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290139648/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879200.761462:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879200.761468:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:3.0:1788879200.761473:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879200.761477:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879200.761480:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879200.761487:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879200.761490:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879200.761496:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879200.761499:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879200.761501:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879200.761536:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879200.761541:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10139 00000100:00080000:3.0:1788879200.761549:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788879200.761556:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879200.761558:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879200.761566:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879200.761575:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879200.761577:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:3.0:1788879200.761580:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879200.761586:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1788879200.761589:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879200.761591:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.761598:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:3.0:1788879200.761605:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879200.761608:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879200.761618:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879200.761619:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304812585 00000100:00000001:3.0:1788879200.761627:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879200.761636:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788879200.761643:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879200.761653:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879200.761657:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879200.761659:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304812585 02000000:00000001:3.0:1788879200.761665:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879200.761668:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1788879200.761674:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.761677:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879200.761682:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879200.761684:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788879200.761691:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879200.761692:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000008:00000001:3.0:1788879200.761700:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879200.761707:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879200.761709:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879200/real 1788879200] req@ffff94e50449fb80 x1875765290139520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879200.761718:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879200.761725:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879200.761731:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788879200.761732:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:1.0:1788879200.761738:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788879200.761739:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879200.761746:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:3.0:1788879200.761747:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.761753:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879200.761759:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879200.761761:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.761763:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879200.761768:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879200.761786:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290139776/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879200.761787:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290139520/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879200.761884:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.761893:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879200.761895:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879200.761900:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.761953:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879200.761953:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.761960:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879200.761963:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.761966:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879200.761973:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879200.761974:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879200.761981:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879200.761984:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879200.761989:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879200.761991:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00080000:3.0:1788879200.761994:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879200.761998:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788879200.762002:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00100000:1.0:1788879200.762009:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290139520:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1788879200.762014:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10139 00000100:00000001:1.0:1788879200.762025:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788879200.762028:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879200.762030:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879200.762037:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1788879200.762038:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879200.762042:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788879200.762044:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879200.762048:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879200.762052:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879200.762054:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879200.762057:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879200.762060:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000020:00000001:3.0:1788879200.762063:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788879200.762068:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1788879200.762070:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788879200.762074:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879200.762077:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879200.762080:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879200.762083:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879200.762085:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762092:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788879200.762096:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879200.762103:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762104:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1788879200.762106:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879200.762109:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879200.762114:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879200.762116:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879200.762121:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788879200.762124:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762125:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879200.762131:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879200.762133:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879200.762135:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788879200.762139:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879200.762144:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879200.762146:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879200.762154:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879200.762158:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879200.762159:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879200.762165:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879200.762167:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879200.762174:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1788879200.762180:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290139648:192.168.203.129@tcp:38:kworker.0 02000000:00000001:3.0:1788879200.762181:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879200.762188:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762193:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879200.762199:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879200.762200:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879200.762205:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879200.762210:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1788879200.762216:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290139904/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879200.762218:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762233:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879200.762234:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879200.762238:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:3.0:1788879200.762240:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762249:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879200.762250:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879200.762274:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879200.762281:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879200.762286:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879200.762289:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762292:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879200.762296:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762303:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304812044 00000100:00000001:1.0:1788879200.762307:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304812585 00000100:00000001:1.0:1788879200.762311:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304812044 00000100:00000001:1.0:1788879200.762314:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304812585 00000100:00000001:1.0:1788879200.762317:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879200.762319:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762321:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879200.762327:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879200/real 1788879200] req@ffff94e50449c380 x1875765290139648/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879200.762338:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879200.762340:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879200.762343:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879200.762345:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879200.762347:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762349:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879200.762351:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879200.762355:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290139648/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879200.762373:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762376:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879200.762379:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762381:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879200.762382:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762384:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879200.762398:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879200.762401:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879200.762404:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879200.762407:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879200.762411:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290139648:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879200.762416:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879200.762418:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879200.762419:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879200.762422:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879200.762425:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879200.762427:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879200.762431:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290139776:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879200.762435:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879200.762437:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879200.762439:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879200.762441:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879200.762446:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762452:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879200.762453:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:1.0:1788879200.762457:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879200.762469:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879200.762474:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879200.762476:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879200.762478:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762481:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879200.762484:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762487:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304812042 00000100:00000001:1.0:1788879200.762490:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304812585 00000100:00000001:1.0:1788879200.762493:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304812042 00000100:00000001:1.0:1788879200.762496:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304812585 00000100:00000001:1.0:1788879200.762499:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879200.762501:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762524:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879200.762529:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879200/real 1788879200] req@ffff94e50449d180 x1875765290139776/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879200.762539:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879200.762541:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879200.762544:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879200.762546:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879200.762558:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762561:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879200.762562:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879200.762567:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290139776/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879200.762575:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762578:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879200.762581:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762582:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879200.762584:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762586:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879200.762598:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879200.762601:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879200.762605:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879200.762608:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879200.762612:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290139776:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879200.762617:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762619:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879200.762621:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879200.762623:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879200.762625:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879200.762627:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879200.762629:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879200.762632:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879200.762634:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879200.762636:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879200.762638:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762640:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879200.762641:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879200.762643:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879200.762645:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879200.762647:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879200.762648:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879200.762650:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879200.762651:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879200.762653:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879200.762655:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762657:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762660:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879200.762662:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762664:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879200.762666:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879200.762668:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879200.762669:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879200.762671:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879200.762673:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879200.762675:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879200.762678:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879200.762681:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290139904:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879200.762686:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879200.762688:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879200.762690:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879200.762692:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879200.762696:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762702:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879200.762703:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000100:00000001:1.0:1788879200.762708:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879200.762721:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879200.762725:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879200.762728:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879200.762730:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762732:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879200.762735:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762739:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304812042 00000100:00000001:1.0:1788879200.762742:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304812585 00000100:00000001:1.0:1788879200.762745:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304812042 00000100:00000001:1.0:1788879200.762748:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304812585 00000100:00000001:1.0:1788879200.762750:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879200.762753:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762755:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879200.762760:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879200/real 1788879200] req@ffff94e50449e680 x1875765290139904/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879200.762769:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879200.762771:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879200.762773:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879200.762775:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879200.762777:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762779:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879200.762781:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879200.762785:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290139904/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879200.762793:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762796:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879200.762798:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762800:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879200.762802:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762804:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879200.762816:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879200.762819:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879200.762823:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879200.762826:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879200.762830:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290139904:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879200.762834:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762836:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879200.762837:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879200.762839:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879200.762841:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879200.762842:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879200.762844:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879200.762846:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879200.762847:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879200.762849:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879200.762851:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762853:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.762856:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879200.762858:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.762860:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879200.762862:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879200.825020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879200.825026:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879200.825029:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879200.825035:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879200.825038:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879200.825042:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.825045:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879200.825047:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879200.825050:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879201.529049:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879201.529058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879201.529065:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879201.529067:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879201.529097:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879201.529100:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879201.529104:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879201.849040:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879201.849050:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879201.849052:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879201.849058:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879201.849064:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879201.849069:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879201.849072:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879201.849075:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879201.849078:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879202.553001:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879202.553009:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879202.553015:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879202.553017:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879202.553039:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879202.553043:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879202.553046:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879202.873026:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879202.873035:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879202.873039:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879202.873048:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879202.873055:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879202.873061:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879202.873065:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879202.873069:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879202.873074:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879203.576992:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879203.576998:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879203.577002:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879203.577005:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879203.577007:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879203.577009:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879203.577012:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879203.577013:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879203.577015:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879203.577016:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879203.577071:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879203.577073:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879203.577076:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879203.896966:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879203.896971:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879203.896973:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879203.896977:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879203.896980:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879203.896983:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879203.896985:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879203.896987:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879203.896989:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879204.601008:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879204.601015:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879204.601021:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879204.601022:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879204.601045:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879204.601048:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879204.601051:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879204.920999:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879204.921005:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879204.921007:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879204.921012:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879204.921015:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879204.921019:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879204.921021:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879204.921023:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879204.921025:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879205.625013:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879205.625021:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879205.625024:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879205.625028:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879205.625033:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879205.625035:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879205.625037:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879205.625040:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879205.625044:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879205.625045:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879205.625101:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879205.625106:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879205.625111:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879205.881046:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879205.881060:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879205.881065:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879205.881070:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879205.881076:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879205.881084:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10144 00000100:00080000:3.0:1788879205.881098:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879205.881101:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879205.881103:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879205.881113:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879205.881117:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879205.881121:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879205.881124:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879205.881127:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879205.881128:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879205.881134:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879205.881137:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879205.881140:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879205.881144:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879205.881148:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881150:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879205.881153:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879205.881166:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879205.881169:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881173:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879205.881176:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879205.881181:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881195:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879205.881206:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290140032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879205.881219:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879205.881224:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881244:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879205.881249:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879205.881253:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879205.881255:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879205.881257:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879205.881261:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879205.881264:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10144 00000100:00080000:3.0:1788879205.881268:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879205.881270:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879205.881272:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879205.881276:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879205.881277:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879205.881279:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879205.881282:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879205.881284:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879205.881286:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879205.881289:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879205.881292:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879205.881294:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879205.881300:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879205.881303:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881305:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879205.881310:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879205.881314:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881317:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879205.881323:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879205.881324:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881327:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879205.881328:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879205.881331:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881336:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.881339:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1788879205.881344:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290140160/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879205.881348:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879205.881354:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879205.881360:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879205.881362:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879205.881365:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881369:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879205.881375:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879205.881377:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879205.881384:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879205.881385:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879205.881397:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879205.881408:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290140032:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1788879205.881410:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879205.881417:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879205.881419:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788879205.881421:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1788879205.881428:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788879205.881428:0:178134:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:1.0:1788879205.881435:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879205.881437:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10144 02000000:00000001:1.0:1788879205.881443:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788879205.881445:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879205.881450:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879205.881453:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881454:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879205.881462:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879205.881464:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879205.881466:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879205.881491:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879205.881494:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879205.881498:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879205.881503:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879205.881506:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879205.881510:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879205.881515:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879205.881517:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788879205.881519:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.881522:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:3.0:1788879205.881524:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879205.881549:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879205.881550:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:3.0:1788879205.881557:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881561:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879205.881569:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879205.881570:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881573:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879205.881575:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879205.881578:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881586:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879205.881593:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290140288/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879205.881604:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879205.881607:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881612:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879205.881619:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1788879205.881620:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879205.881631:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879205.881633:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879205.881635:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879205.881640:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879205.881646:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10144 00000100:00000001:1.0:1788879205.881648:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:3.0:1788879205.881652:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879205.881657:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879205.881661:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879205.881662:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879205.881671:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879205.881671:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879205.881675:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879205.881680:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879205.881683:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:3.0:1788879205.881686:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879205.881692:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879205.881695:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879205.881699:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879205.881701:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879205.881711:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.881713:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304817705 02000000:00000001:3.0:1788879205.881714:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879205.881721:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879205.881726:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881728:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879205.881731:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879205.881733:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00000001:3.0:1788879205.881748:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881751:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.881759:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304817705 02000000:00000001:3.0:1788879205.881762:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879205.881767:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.881771:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879205.881771:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879205.881776:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879205.881782:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.881783:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879205.881791:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881794:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.881797:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879205.881808:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290140416/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879205.881820:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879205/real 1788879205] req@ffff94e50449df80 x1875765290140032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879205.881833:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879205.881841:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879205.881851:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879205.881867:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879205.881872:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879205.881880:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879205.881886:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879205.881891:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.881896:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879205.881900:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879205.881969:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290140032/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879205.881980:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.881984:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879205.881988:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.881990:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879205.881992:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.881995:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879205.881997:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879205.882000:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879205.882005:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879205.882008:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879205.882012:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290140032:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879205.882017:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882020:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879205.882022:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879205.882024:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879205.882028:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879205.882030:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879205.882033:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879205.882038:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879205.882040:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879205.882043:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879205.882045:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882047:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882055:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879205.882057:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.882060:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879205.882062:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879205.882064:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879205.882066:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879205.882068:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879205.882070:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879205.882073:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879205.882076:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879205.882080:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290140160:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879205.882084:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879205.882086:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879205.882088:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879205.882090:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879205.882094:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.882103:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879205.882104:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:1.0:1788879205.882109:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879205.882128:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879205.882134:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879205.882136:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879205.882138:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.882140:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879205.882143:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.882148:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304817164 00000100:00000001:1.0:1788879205.882152:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304817705 00000100:00000001:1.0:1788879205.882154:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304817164 00000100:00000001:1.0:1788879205.882157:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304817705 00000100:00000001:1.0:1788879205.882160:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879205.882162:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.882164:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879205.882169:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879205/real 1788879205] req@ffff94e50449ce00 x1875765290140160/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879205.882179:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879205.882180:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879205.882183:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879205.882185:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879205.882186:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882188:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879205.882190:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879205.882194:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290140160/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879205.882212:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882215:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879205.882218:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882220:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879205.882221:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882223:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879205.882236:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879205.882239:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879205.882242:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879205.882245:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879205.882248:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290140160:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879205.882253:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879205.882254:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879205.882256:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879205.882259:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879205.882261:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879205.882263:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879205.882266:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290140288:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879205.882269:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879205.882271:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879205.882273:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879205.882274:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879205.882278:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.882283:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879205.882285:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000100:00000001:1.0:1788879205.882288:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879205.882299:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879205.882304:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879205.882306:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879205.882307:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.882310:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879205.882312:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.882316:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304817162 00000100:00000001:1.0:1788879205.882319:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304817705 00000100:00000001:1.0:1788879205.882321:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304817162 00000100:00000001:1.0:1788879205.882324:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304817705 00000100:00000001:1.0:1788879205.882326:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879205.882328:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.882330:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879205.882334:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879205/real 1788879205] req@ffff94e50449e680 x1875765290140288/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879205.882343:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879205.882344:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879205.882346:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879205.882348:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879205.882349:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882351:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879205.882352:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879205.882356:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290140288/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879205.882364:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882369:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879205.882372:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882374:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879205.882376:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882378:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879205.882389:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879205.882392:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879205.882395:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879205.882397:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879205.882402:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290140288:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879205.882406:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879205.882407:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879205.882409:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879205.882411:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879205.882413:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879205.882415:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879205.882418:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290140416:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879205.882421:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879205.882423:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879205.882424:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879205.882426:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879205.882429:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.882434:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879205.882435:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:1.0:1788879205.882439:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879205.882448:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879205.882452:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879205.882455:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879205.882456:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.882458:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879205.882461:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.882464:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304817162 00000100:00000001:1.0:1788879205.882467:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304817705 00000100:00000001:1.0:1788879205.882494:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304817162 00000100:00000001:1.0:1788879205.882497:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304817705 00000100:00000001:1.0:1788879205.882500:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879205.882502:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.882505:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879205.882510:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879205/real 1788879205] req@ffff94e50449d180 x1875765290140416/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879205.882546:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879205.882548:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879205.882552:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879205.882554:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879205.882556:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882558:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879205.882560:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879205.882565:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290140416/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879205.882578:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882582:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879205.882586:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882590:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879205.882593:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882596:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879205.882615:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879205.882621:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879205.882628:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879205.882633:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879205.882640:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290140416:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879205.882648:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882652:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879205.882655:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879205.882659:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879205.882663:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879205.882666:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879205.882669:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879205.882674:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879205.882676:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879205.882679:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879205.882681:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882684:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879205.882685:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879205.882687:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879205.882689:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879205.882691:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879205.882693:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879205.882695:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879205.882696:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879205.882698:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879205.882699:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882701:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879205.882703:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879205.882704:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879205.882706:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879205.882707:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879205.882708:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879205.882710:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879205.882711:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879205.882713:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879205.882714:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882716:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.882721:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879205.882723:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.882725:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879205.882727:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879205.945006:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879205.945014:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879205.945017:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879205.945024:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879205.945029:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879205.945032:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.945035:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879205.945039:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879205.945042:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879206.649024:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879206.649029:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879206.649033:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879206.649034:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879206.649050:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879206.649052:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879206.649055:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879206.968973:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879206.968980:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879206.968981:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879206.968986:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879206.968990:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879206.968993:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879206.968995:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879206.968998:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879206.969001:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879207.673014:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879207.673019:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879207.673023:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879207.673024:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879207.673065:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879207.673068:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879207.673071:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879207.992978:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879207.992984:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879207.992986:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879207.992990:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879207.992994:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879207.992996:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879207.992998:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879207.993000:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879207.993002:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879208.697020:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879208.697027:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879208.697032:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879208.697033:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879208.697055:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879208.697058:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879208.697061:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879209.016992:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879209.017000:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879209.017002:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879209.017007:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879209.017012:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879209.017016:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879209.017019:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879209.017022:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879209.017025:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879209.721017:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879209.721023:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879209.721027:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879209.721028:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879209.721062:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879209.721069:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879209.721071:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879210.040969:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879210.040976:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879210.040977:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879210.040982:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879210.040986:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879210.040989:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879210.040991:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879210.040993:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879210.040995:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879210.744998:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879210.745004:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879210.745009:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879210.745011:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879210.745058:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879210.745061:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879210.745064:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879211.000995:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879211.001009:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879211.001011:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879211.001014:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879211.001018:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879211.001024:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10149 00000100:00080000:0.0:1788879211.001032:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879211.001034:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879211.001036:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879211.001045:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879211.001048:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879211.001051:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879211.001054:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879211.001056:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879211.001058:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879211.001063:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879211.001066:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879211.001069:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879211.001072:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879211.001075:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879211.001078:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879211.001080:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879211.001089:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879211.001090:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879211.001093:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879211.001095:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879211.001099:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879211.001108:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879211.001116:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290140544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879211.001124:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879211.001128:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879211.001140:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879211.001142:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879211.001145:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879211.001146:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879211.001147:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879211.001149:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879211.001151:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10149 00000100:00080000:0.0:1788879211.001154:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879211.001155:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879211.001157:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879211.001159:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879211.001161:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879211.001163:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879211.001164:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879211.001166:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879211.001167:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879211.001169:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879211.001171:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001171:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788879211.001176:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879211.001177:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879211.001179:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1788879211.001181:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879211.001182:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879211.001185:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879211.001186:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879211.001190:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879211.001191:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:0.0:1788879211.001193:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879211.001194:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1788879211.001197:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879211.001199:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879211.001200:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879211.001204:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1788879211.001206:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290140544:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788879211.001207:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879211.001211:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879211.001213:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879211.001214:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879211.001218:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879211.001218:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879211.001221:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879211.001222:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879211.001223:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879211.001229:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879211.001230:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290140672/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879211.001238:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879211.001239:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879211.001241:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001245:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879211.001248:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00080000:0.0:1788879211.001250:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879211.001251:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788879211.001255:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879211.001257:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879211.001259:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879211.001261:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879211.001263:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10149 00000100:00080000:0.0:1788879211.001276:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879211.001277:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879211.001278:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879211.001280:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879211.001281:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879211.001285:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000800:00000100:1.0:1788879211.001285:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:0.0:1788879211.001289:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879211.001292:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879211.001294:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879211.001297:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879211.001300:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001300:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:0.0:1788879211.001304:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879211.001307:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1788879211.001308:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879211.001312:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001313:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879211.001314:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879211.001317:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879211.001318:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000008:00000001:0.0:1788879211.001320:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879211.001323:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001326:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879211.001327:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879211.001330:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879211.001332:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879211.001333:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879211.001334:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304822825 02000000:00000001:0.0:1788879211.001335:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879211.001342:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879211.001346:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290140800/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879211.001349:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788879211.001355:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879211.001358:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001359:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304822825 00000100:00000001:0.0:1788879211.001360:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879211.001365:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879211.001365:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:0.0:1788879211.001371:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879211.001371:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879211.001376:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879211.001376:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879211.001378:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879211.001380:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879211.001381:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879211.001384:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10149 00000100:00100000:1.0:1788879211.001388:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879211/real 1788879211] req@ffff94e508979c00 x1875765290140544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879211.001389:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879211.001393:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879211.001395:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879211.001400:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879211.001402:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879211.001405:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879211.001407:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000400:00000001:1.0:1788879211.001407:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879211.001410:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879211.001411:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879211.001414:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879211.001415:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:0.0:1788879211.001418:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879211.001419:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879211.001423:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001423:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879211.001428:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879211.001429:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879211.001431:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879211.001432:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879211.001451:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879211.001451:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290140544/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879211.001471:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879211.001474:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879211.001481:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001483:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879211.001484:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001488:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879211.001490:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.001494:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879211.001495:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879211.001497:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879211.001498:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:0.0:1788879211.001501:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001502:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879211.001505:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:1.0:1788879211.001507:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:0.0:1788879211.001508:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879211.001512:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879211.001512:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879211.001517:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1788879211.001518:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879211.001522:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290140544:192.168.203.129@tcp:250:kworker.0 00000100:00100000:0.0:1788879211.001528:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290140928/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879211.001529:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.001534:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879211.001537:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1788879211.001541:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879211.001541:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879211.001544:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1788879211.001545:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879211.001548:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:0.0:1788879211.001550:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879211.001551:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879211.001573:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879211.001576:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879211.001580:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879211.001582:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.001586:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.001595:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879211.001599:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001603:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879211.001606:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879211.001608:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879211.001610:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879211.001612:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879211.001616:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879211.001619:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879211.001623:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879211.001629:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290140672:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879211.001635:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879211.001639:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879211.001642:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879211.001645:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879211.001650:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001661:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879211.001664:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:1.0:1788879211.001670:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879211.001702:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879211.001715:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879211.001717:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879211.001718:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001719:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879211.001721:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001725:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304822283 00000100:00000001:1.0:1788879211.001727:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304822825 00000100:00000001:1.0:1788879211.001729:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304822283 00000100:00000001:1.0:1788879211.001731:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304822825 00000100:00000001:1.0:1788879211.001733:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879211.001734:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001735:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879211.001739:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879211/real 1788879211] req@ffff94e50897ad80 x1875765290140672/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879211.001744:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879211.001745:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879211.001747:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879211.001749:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879211.001750:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.001751:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879211.001752:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879211.001755:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290140672/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879211.001760:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.001761:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879211.001764:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.001765:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879211.001766:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.001767:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879211.001776:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879211.001779:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879211.001781:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879211.001783:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879211.001785:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290140672:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879211.001787:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879211.001788:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879211.001789:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879211.001791:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879211.001792:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879211.001793:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879211.001795:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290140800:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879211.001798:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879211.001799:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879211.001801:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879211.001802:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879211.001804:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001807:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879211.001808:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:1.0:1788879211.001811:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879211.001818:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879211.001821:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879211.001822:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879211.001824:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001825:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879211.001827:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001829:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304822281 00000100:00000001:1.0:1788879211.001831:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304822825 00000100:00000001:1.0:1788879211.001833:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304822281 00000100:00000001:1.0:1788879211.001835:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304822825 00000100:00000001:1.0:1788879211.001837:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879211.001838:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001839:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879211.001842:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879211/real 1788879211] req@ffff94e508978e00 x1875765290140800/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879211.001847:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879211.001848:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879211.001849:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879211.001850:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879211.001851:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.001853:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879211.001854:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879211.001873:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290140800/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879211.001877:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.001878:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879211.001879:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.001880:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879211.001881:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.001882:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879211.001890:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879211.001892:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879211.001893:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879211.001895:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879211.001897:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290140800:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879211.001899:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879211.001899:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879211.001901:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879211.001902:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879211.001903:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879211.001905:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879211.001906:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290140928:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879211.001908:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879211.001909:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879211.001910:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879211.001911:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879211.001913:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001916:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879211.001916:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:1.0:1788879211.001944:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879211.001953:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879211.001957:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879211.001958:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879211.001959:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001960:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879211.001962:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001964:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304822282 00000100:00000001:1.0:1788879211.001966:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304822825 00000100:00000001:1.0:1788879211.001968:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304822282 00000100:00000001:1.0:1788879211.001969:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304822825 00000100:00000001:1.0:1788879211.001971:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879211.001972:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.001973:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879211.001976:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879211/real 1788879211] req@ffff94e50897bb80 x1875765290140928/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879211.001981:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879211.001981:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879211.001983:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879211.001984:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879211.001985:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.001986:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879211.001987:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879211.001990:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290140928/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879211.001994:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.001995:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879211.001997:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.001998:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879211.001998:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.002000:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879211.002008:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879211.002010:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879211.002011:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879211.002012:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879211.002014:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290140928:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879211.002016:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.002018:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879211.002018:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879211.002020:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879211.002021:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879211.002022:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879211.002023:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879211.002025:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879211.002026:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879211.002028:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879211.002029:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.002030:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879211.002031:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879211.002032:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879211.002033:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879211.002034:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879211.002035:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879211.002036:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879211.002036:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879211.002037:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879211.002039:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.002040:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879211.002041:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879211.002041:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879211.002043:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879211.002043:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879211.002044:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879211.002046:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879211.002046:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879211.002047:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879211.002048:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.002049:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.002051:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879211.002052:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.002053:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879211.002054:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879211.064956:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879211.064964:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879211.064967:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879211.064972:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879211.064975:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879211.064977:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.064980:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879211.064981:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879211.064983:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879211.769037:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879211.769047:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879211.769055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879211.769057:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879211.769097:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879211.769102:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879211.769106:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879212.089004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879212.089012:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879212.089014:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879212.089020:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879212.089025:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879212.089029:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879212.089032:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879212.089036:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879212.089039:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879212.792977:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879212.792983:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879212.792987:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879212.792988:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879212.793022:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879212.793024:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879212.793026:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879213.113033:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879213.113043:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879213.113047:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879213.113056:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879213.113062:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879213.113068:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879213.113073:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879213.113077:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879213.113082:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879213.816998:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879213.817004:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879213.817008:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879213.817022:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879213.817025:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879213.817028:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879213.817030:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879213.817032:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879213.817033:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879213.817035:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879213.817072:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879213.817073:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879213.817087:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879214.137084:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879214.137094:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879214.137097:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879214.137105:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879214.137111:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879214.137115:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879214.137119:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879214.137123:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879214.137126:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879214.841064:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879214.841075:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879214.841085:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879214.841088:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879214.841160:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879214.841163:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879214.841167:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879215.161025:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879215.161034:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879215.161036:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879215.161043:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879215.161048:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879215.161052:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879215.161056:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879215.161070:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879215.161074:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879215.865017:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879215.865025:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879215.865028:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879215.865032:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879215.865038:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879215.865041:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879215.865044:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879215.865047:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879215.865051:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879215.865064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879215.865121:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879215.865126:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879215.865131:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879216.121019:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879216.121029:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879216.121031:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879216.121035:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879216.121040:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879216.121046:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10154 00000100:00080000:2.0:1788879216.121057:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879216.121061:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879216.121064:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879216.121075:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879216.121079:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879216.121084:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879216.121088:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879216.121091:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879216.121093:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879216.121100:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879216.121105:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879216.121109:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879216.121114:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879216.121118:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879216.121122:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879216.121125:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879216.121139:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879216.121142:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879216.121146:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879216.121150:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879216.121166:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879216.121180:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879216.121195:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4000 x1875765290141056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879216.121210:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879216.121216:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879216.121237:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879216.121243:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879216.121272:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879216.121274:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879216.121277:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879216.121283:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879216.121288:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10154 00000100:00080000:2.0:1788879216.121295:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879216.121300:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879216.121302:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879216.121304:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879216.121310:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788879216.121313:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879216.121314:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788879216.121317:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879216.121320:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1788879216.121323:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879216.121324:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879216.121330:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:1.0:1788879216.121344:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879216.121345:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879216.121351:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879216.121352:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879216.121362:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879216.121371:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290141056:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1788879216.121375:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879216.121380:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788879216.121382:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879216.121388:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879216.121388:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879216.121395:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879216.121417:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879216.121418:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879216.121426:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879216.121429:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879216.121441:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879216.121447:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879216.121452:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879216.121454:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.121458:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788879216.121462:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879216.121463:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074548224 : -117763635003392 : ffff94e5045e4000) 02000000:00000001:2.0:1788879216.121467:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.121471:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879216.121473:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879216.121477:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879216.121480:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879216.121490:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879216.121498:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290141184/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879216.121507:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879216.121510:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879216.121516:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879216.121519:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879216.121523:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879216.121525:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879216.121527:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879216.121530:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879216.121533:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879216.121534:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10154 00000100:00000001:1.0:1788879216.121543:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304827945 00000100:00080000:2.0:1788879216.121545:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879216.121552:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879216.121553:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879216.121559:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1788879216.121560:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879216.121564:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879216.121569:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879216.121571:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304827945 00000020:00000001:2.0:1788879216.121573:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879216.121579:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879216.121579:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879216.121587:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879216.121588:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879216.121595:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879216.121596:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879216.121602:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879216.121603:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.121610:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879216.121611:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879216.121619:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879216.121623:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1788879216.121624:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.121628:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788879216.121630:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879216.121636:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:2.0:1788879216.121637:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879216.121640:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00000001:2.0:1788879216.121644:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.121645:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879216.121649:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.121650:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879216.121654:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879216.121658:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879216.121659:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879216.121665:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879216.121666:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879216.121672:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879216.121673:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879216.121676:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879216.121682:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879216.121683:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290141184:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879216.121690:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788879216.121692:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879216.121693:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879216.121698:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879216.121702:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1788879216.121707:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290141312/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879216.121710:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.121722:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879216.121723:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879216.121726:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074559872 : -117763634991744 : ffff94e5045e6d80) 00000100:00000001:2.0:1788879216.121729:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.121735:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879216.121738:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879216.121743:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879216.121748:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879216.121750:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879216.121753:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879216.121754:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879216.121763:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879216.121765:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304827403 00000100:00080000:2.0:1788879216.121771:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10154 00000100:00000001:1.0:1788879216.121773:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304827945 00000100:00000001:1.0:1788879216.121780:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304827403 00000100:00080000:2.0:1788879216.121781:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879216.121783:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304827945 00000100:00000001:2.0:1788879216.121786:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879216.121790:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879216.121791:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879216.121796:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879216.121799:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879216.121802:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879216.121803:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879216.121806:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788879216.121808:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879216.121811:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:2.0:1788879216.121814:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879216.121816:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788879216.121819:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879216.121821:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879216.121824:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879216.121826:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879216.121830:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1788879216.121835:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879216.121837:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879216.121842:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.121843:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879216.121848:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879216.121852:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879216.121853:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879216.121860:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879216.121862:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879216.121871:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879216.121872:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879216.121878:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879216.121902:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290141312:192.168.203.129@tcp:8:kworker.0 00000008:00000001:2.0:1788879216.121903:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879216.121911:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.121915:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879216.121963:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879216.121966:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879216.121970:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879216.121976:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879216.121976:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879216.121982:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879216.121985:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879216.121989:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879216.121998:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879216.122006:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879216.122011:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290141440/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879216.122020:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879216.122025:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:2.0:1788879216.122033:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879216.122035:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879216.122042:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879216.122051:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.122075:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.122088:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304827402 00000100:00000001:1.0:1788879216.122093:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304827945 00000100:00000001:1.0:1788879216.122096:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304827402 00000100:00000001:1.0:1788879216.122099:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304827945 00000100:00000001:1.0:1788879216.122103:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879216.122106:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.122110:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.122113:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879216.122119:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879216.122122:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788879216.122124:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879216.122127:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879216.122129:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879216.122131:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879216.122133:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879216.122136:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879216.122139:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879216.122143:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879216.122149:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290141440:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879216.122154:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879216.122156:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879216.122158:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879216.122161:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879216.122168:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.122178:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879216.122180:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00000001:1.0:1788879216.122185:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879216.122201:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.122206:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304827402 00000100:00000001:1.0:1788879216.122209:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304827945 00000100:00000001:1.0:1788879216.122212:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304827402 00000100:00000001:1.0:1788879216.122216:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304827945 00000100:00000001:1.0:1788879216.122219:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879216.122222:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.122225:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.122228:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879216.122231:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879216.122233:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788879216.122236:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879216.122237:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879216.122240:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879216.122243:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879216.184997:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879216.185005:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879216.185007:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879216.185013:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879216.185016:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879216.185020:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879216.185023:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879216.185025:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879216.185028:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879216.889034:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879216.889043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879216.889049:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879216.889051:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879216.889105:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879216.889110:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879216.889116:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879217.208989:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879217.208996:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879217.208999:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879217.209004:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879217.209008:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879217.209012:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879217.209015:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879217.209017:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879217.209020:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879217.913029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879217.913035:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879217.913040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879217.913041:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879217.913089:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879217.913092:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879217.913094:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879218.233027:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879218.233036:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879218.233038:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879218.233062:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879218.233066:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879218.233070:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879218.233073:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879218.233076:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879218.233079:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879218.937111:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879218.937125:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879218.937136:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879218.937138:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879218.937186:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879218.937190:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879218.937194:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788879219.193420:0:2352:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1788879219.193488:0:2352:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788879219.193513:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879219.193536:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879219.193560:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879219.193564:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879219.193568:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879219.193587:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:1.0:1788879219.193603:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879219.193610:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879219.193618:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879219.193619:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879219.193634:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879219.193642:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879219.193643:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879216/real 1788879219] req@ffff94e5045e4000 x1875765290141056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879219.193654:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000400:00000001:1.0:1788879219.193672:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879219.193675:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000800:00000100:0.0:1788879219.193681:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:1.0:1788879219.193687:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879219.193694:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879219.193698:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879219.193698:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.193706:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879219.193709:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879219.193710:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1788879219.193720:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4000 x1875765290141056/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879219.193722:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879219.193739:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879219.193742:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.193747:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879219.193750:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.193756:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879219.193760:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000800:00000100:0.0:1788879219.193762:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:1.0:1788879219.193766:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879219.193774:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879219.193779:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:1.0:1788879219.193781:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1788879219.193790:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1788879219.193793:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879219.193797:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879219.193800:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1788879219.193805:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:1.0:1788879219.193815:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290141056:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879219.193827:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879219.193831:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879219.193834:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879219.193837:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879219.193845:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879219.193849:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879219.193851:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879219.193854:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879219.193856:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879219.193860:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879219.193861:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.193864:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879219.193866:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879219.193867:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879219.193868:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879219.193874:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879216/real 1788879219] req@ffff94e5045e7b80 x1875765290141440/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879219.193883:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879219.193885:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879219.193889:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879219.193891:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879219.193892:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.193894:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879219.193896:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879219.193899:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290141440/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879219.193933:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.193936:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879219.193938:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.193940:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879219.193941:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.193943:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879219.193959:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879219.193961:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879219.193965:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879219.193968:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879219.193971:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290141440:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879219.193976:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879219.193980:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879216/real 1788879219] req@ffff94e5045e4e00 x1875765290141312/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879219.193989:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879219.193990:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879219.193993:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879219.193994:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879219.193995:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.193997:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879219.193999:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879219.194002:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290141312/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879219.194008:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.194010:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879219.194012:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.194013:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879219.194015:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.194016:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879219.194029:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879219.194031:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879219.194035:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879219.194037:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879219.194041:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290141312:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879219.194045:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879219.194049:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879216/real 1788879219] req@ffff94e5045e6d80 x1875765290141184/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879219.194056:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879219.194058:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879219.194061:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879219.194062:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879219.194064:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.194066:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879219.194067:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879219.194071:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6d80 x1875765290141184/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879219.194078:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.194080:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879219.194082:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.194084:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879219.194085:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.194087:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879219.194101:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879219.194103:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879219.194107:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879219.194109:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879219.194113:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290141184:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879219.194117:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.194119:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879219.194120:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879219.194122:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879219.194125:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879219.194127:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879219.194128:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879219.194130:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879219.194132:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879219.194134:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879219.194135:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.194137:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879219.194138:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879219.194140:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879219.194142:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879219.194143:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879219.194144:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879219.194146:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879219.194147:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879219.194149:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879219.194151:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.194153:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879219.194154:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879219.194155:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879219.194157:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879219.194158:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879219.194159:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879219.194161:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879219.194162:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879219.194165:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879219.194166:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.194168:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879219.194176:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879219.194178:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879219.194181:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879219.194183:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879219.257008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879219.257018:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879219.257021:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879219.257029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879219.257034:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879219.257040:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879219.257044:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879219.257048:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879219.257051:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879219.961046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879219.961059:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879219.961070:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879219.961072:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879219.961157:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879219.961163:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879219.961167:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879220.281087:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879220.281099:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879220.281102:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879220.281110:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879220.281115:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879220.281121:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879220.281124:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879220.281128:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879220.281131:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879220.985086:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879220.985095:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879220.985102:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879220.985103:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879220.985178:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879220.985182:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879220.985186:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879221.241131:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879221.241147:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879221.241153:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879221.241159:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879221.241166:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879221.241179:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10160 00000100:00080000:3.0:1788879221.241196:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879221.241200:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879221.241203:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879221.241217:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879221.241221:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879221.241227:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879221.241231:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879221.241234:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879221.241236:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879221.241242:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879221.241245:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879221.241249:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879221.241254:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879221.241259:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879221.241262:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879221.241265:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879221.241282:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879221.241286:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879221.241293:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879221.241298:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879221.241307:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879221.241331:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879221.241351:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290141568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879221.241395:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879221.241405:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879221.241446:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879221.241453:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879221.241460:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879221.241463:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879221.241467:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879221.241473:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879221.241480:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10160 00000100:00080000:3.0:1788879221.241488:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879221.241493:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879221.241496:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879221.241504:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879221.241509:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879221.241514:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879221.241520:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879221.241524:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879221.241527:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879221.241536:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879221.241541:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879221.241545:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879221.241555:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788879221.241558:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879221.241568:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.241569:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879221.241575:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879221.241578:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879221.241586:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1788879221.241587:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879221.241591:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788879221.241599:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879221.241605:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879221.241608:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879221.241615:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879221.241623:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879221.241639:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290141568:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788879221.241640:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879221.241646:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.241655:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879221.241656:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879221.241664:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879221.241668:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879221.241670:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879221.241677:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879221.241685:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879221.241688:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879221.241698:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879221.241707:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290141696/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879221.241729:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879221.241733:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.241738:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879221.241741:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.241747:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00080000:3.0:1788879221.241751:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879221.241760:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879221.241763:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879221.241769:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879221.241773:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879221.241779:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879221.241785:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10160 00000100:00080000:3.0:1788879221.241795:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879221.241801:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879221.241804:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879221.241810:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879221.241814:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879221.241819:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879221.241824:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879221.241828:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879221.241831:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879221.241838:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879221.241849:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879221.241850:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879221.241857:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879221.241868:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879221.241873:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879221.241877:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879221.241880:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00100000:3.0:1788879221.241884:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879221.241892:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00000001:3.0:1788879221.241894:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.241901:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879221.241963:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879221.241976:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879221.241979:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879221.241983:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879221.241988:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879221.242002:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879221.242006:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.242020:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879221.242022:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879221.242039:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290141824/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879221.242056:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304833066 00000100:00000001:3.0:1788879221.242059:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879221.242069:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879221.242075:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879221.242082:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879221.242089:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879221.242092:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879221.242093:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879221.242100:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879221.242111:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879221.242112:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304833066 00000100:00080000:3.0:1788879221.242119:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10160 00000100:00000001:1.0:1788879221.242127:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788879221.242132:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879221.242138:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879221.242141:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879221.242150:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879221.242151:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879221.242163:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879221.242164:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1788879221.242172:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879221.242179:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:1.0:1788879221.242184:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879221/real 1788879221] req@ffff94e50449c380 x1875765290141568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788879221.242187:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879221.242199:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879221.242205:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879221.242222:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000400:00000001:1.0:1788879221.242228:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788879221.242230:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879221.242235:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788879221.242239:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879221.242249:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879221.242258:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879221.242262:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.242268:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879221.242272:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879221.242284:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290141568/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879221.242313:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.242320:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879221.242326:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.242330:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879221.242332:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.242337:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879221.242341:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879221.242348:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879221.242356:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879221.242360:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879221.242396:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290141568:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879221.242406:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.242410:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879221.242414:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879221.242419:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879221.242426:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879221.242430:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879221.242435:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879221.242442:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879221.242445:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879221.242450:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879221.242453:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.242457:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.242471:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879221.242475:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.242480:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879221.242482:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879221.242485:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879221.242489:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879221.242491:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879221.242496:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879221.242500:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879221.242504:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879221.242512:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290141696:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879221.242519:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879221.242524:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879221.242528:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879221.242533:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879221.242539:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.242553:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879221.242557:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 02000000:00000001:3.0:1788879221.242566:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879221.242568:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788879221.242579:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879221.242585:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879221.242590:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879221.242596:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879221.242601:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879221.242609:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879221.242616:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879221.242622:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.242624:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879221.242628:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879221.242634:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788879221.242635:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879221.242642:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879221.242644:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.242651:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879221.242662:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.242663:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.242677:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304832524 00000100:00100000:3.0:1788879221.242684:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290141952/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879221.242686:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304833066 00000100:00000001:1.0:1788879221.242697:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304832524 00000100:00000001:1.0:1788879221.242707:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304833066 00000100:00000001:1.0:1788879221.242717:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879221.242720:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879221.242725:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879221.242731:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.242735:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879221.242745:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879221.242754:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879221/real 1788879221] req@ffff94e50449fb80 x1875765290141696/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879221.242774:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879221.242777:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879221.242784:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879221.242790:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879221.242793:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.242797:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879221.242800:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879221.242810:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290141696/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879221.242837:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.242842:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879221.242847:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.242850:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879221.242854:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.242858:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879221.242897:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879221.242942:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879221.242951:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879221.242956:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879221.242964:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290141696:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879221.242972:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879221.242975:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879221.242978:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879221.242982:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879221.242986:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879221.242990:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879221.242996:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290141824:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879221.243002:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879221.243006:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879221.243009:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879221.243013:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879221.243020:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.243032:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879221.243035:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:1.0:1788879221.243042:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879221.243071:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879221.243081:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879221.243086:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879221.243089:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.243093:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879221.243099:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.243107:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304832523 00000100:00000001:1.0:1788879221.243112:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304833066 00000100:00000001:1.0:1788879221.243117:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304832523 00000100:00000001:1.0:1788879221.243121:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304833066 00000100:00000001:1.0:1788879221.243126:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879221.243130:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.243134:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879221.243143:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879221/real 1788879221] req@ffff94e50449d880 x1875765290141824/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879221.243163:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879221.243165:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879221.243170:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879221.243174:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879221.243177:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.243181:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879221.243184:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879221.243192:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290141824/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879221.243206:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.243210:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879221.243215:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.243218:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879221.243221:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.243225:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879221.243249:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879221.243255:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879221.243262:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879221.243266:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879221.243274:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290141824:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879221.243282:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.243286:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879221.243288:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879221.243291:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879221.243296:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879221.243299:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879221.243301:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879221.243306:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879221.243309:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879221.243313:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879221.243315:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.243318:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879221.243320:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879221.243322:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879221.243326:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879221.243328:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879221.243330:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879221.243333:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879221.243335:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879221.243339:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879221.243341:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.243344:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.243352:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879221.243355:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.243359:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879221.243389:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879221.243393:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879221.243396:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879221.243399:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879221.243404:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879221.243407:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879221.243411:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879221.243418:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290141952:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879221.243425:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879221.243428:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879221.243431:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879221.243435:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879221.243442:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.243452:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879221.243455:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:1.0:1788879221.243462:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879221.243505:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879221.243515:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879221.243519:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879221.243522:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.243525:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879221.243530:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.243538:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304832523 00000100:00000001:1.0:1788879221.243542:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304833066 00000100:00000001:1.0:1788879221.243547:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304832523 00000100:00000001:1.0:1788879221.243551:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304833066 00000100:00000001:1.0:1788879221.243554:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879221.243558:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.243561:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879221.243570:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879221/real 1788879221] req@ffff94e50449f800 x1875765290141952/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879221.243587:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879221.243589:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879221.243594:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879221.243597:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879221.243600:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.243604:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879221.243607:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879221.243615:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290141952/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879221.243628:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.243632:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879221.243637:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.243640:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879221.243643:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.243647:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879221.243670:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879221.243675:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879221.243682:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879221.243687:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879221.243694:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290141952:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879221.243700:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.243704:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879221.243706:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879221.243709:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879221.243714:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879221.243717:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879221.243719:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879221.243723:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879221.243725:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879221.243729:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879221.243731:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.243735:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879221.243741:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879221.243743:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879221.243747:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879221.243750:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879221.305053:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879221.305062:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879221.305065:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879221.305072:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879221.305077:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879221.305082:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879221.305086:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879221.305090:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879221.305094:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879222.009057:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879222.009065:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879222.009071:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879222.009073:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879222.009111:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879222.009113:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879222.009117:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879222.329057:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879222.329064:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879222.329067:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879222.329073:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879222.329077:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879222.329081:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879222.329085:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879222.329088:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879222.329091:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879223.033023:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879223.033029:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879223.033033:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879223.033036:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879223.033038:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879223.033040:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879223.033043:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879223.033045:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879223.033046:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879223.033048:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879223.033094:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879223.033096:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879223.033098:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879223.353079:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879223.353088:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879223.353091:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879223.353097:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879223.353101:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879223.353105:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879223.353109:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879223.353112:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879223.353116:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879224.056985:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879224.056992:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879224.056999:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879224.057001:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879224.057024:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879224.057027:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879224.057030:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879224.377065:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879224.377076:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879224.377080:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879224.377101:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879224.377107:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879224.377114:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879224.377118:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879224.377123:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879224.377127:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879225.081061:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879225.081074:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879225.081079:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879225.081084:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879225.081092:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879225.081096:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879225.081101:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879225.081105:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879225.081113:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879225.081116:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879225.081222:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879225.081228:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879225.081235:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879225.400967:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879225.400973:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879225.400974:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879225.400979:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879225.400982:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879225.400985:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879225.400987:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879225.400989:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879225.400991:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879226.105095:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879226.105103:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879226.105110:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879226.105112:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879226.105138:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879226.105141:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879226.105145:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879226.361010:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879226.361020:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879226.361022:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879226.361026:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879226.361029:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879226.361035:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10165 00000100:00080000:3.0:1788879226.361044:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879226.361046:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879226.361047:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879226.361054:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879226.361056:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879226.361060:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879226.361062:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879226.361064:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879226.361065:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879226.361069:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879226.361072:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879226.361074:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879226.361077:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879226.361080:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361081:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879226.361083:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879226.361095:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879226.361097:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361101:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879226.361103:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879226.361108:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361121:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879226.361131:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290142080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879226.361141:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879226.361144:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361157:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879226.361160:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879226.361162:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879226.361164:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879226.361165:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879226.361167:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879226.361169:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10165 00000100:00080000:3.0:1788879226.361171:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879226.361173:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879226.361174:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879226.361176:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879226.361177:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879226.361179:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879226.361180:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879226.361186:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879226.361187:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879226.361189:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879226.361191:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879226.361193:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879226.361197:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879226.361199:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361200:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879226.361203:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879226.361206:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361208:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879226.361211:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879226.361213:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361214:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879226.361215:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879226.361217:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361218:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879226.361221:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361223:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879226.361226:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00100000:3.0:1788879226.361228:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290142208/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879226.361229:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879226.361232:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879226.361238:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879226.361241:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879226.361247:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361252:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879226.361255:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361257:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879226.361259:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788879226.361260:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290142080:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1788879226.361261:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879226.361264:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879226.361268:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879226.361269:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879226.361272:0:136860:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:1.0:1788879226.361274:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1788879226.361277:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10165 02000000:00000001:1.0:1788879226.361278:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879226.361281:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788879226.361282:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879226.361286:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879226.361288:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361288:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879226.361293:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879226.361295:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879226.361297:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879226.361298:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879226.361299:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879226.361301:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879226.361303:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879226.361304:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879226.361305:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361307:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 02000000:00000001:3.0:1788879226.361308:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879226.361312:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879226.361314:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361324:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879226.361340:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:3.0:1788879226.361341:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879226.361347:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361348:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879226.361354:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879226.361355:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361358:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879226.361360:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879226.361362:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361369:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879226.361375:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1788879226.361377:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290142336/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879226.361387:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879226.361388:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879226.361391:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361392:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879226.361395:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361396:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879226.361399:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879226.361400:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:3.0:1788879226.361405:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879226.361406:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361407:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879226.361410:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304838185 00000100:00080000:3.0:1788879226.361411:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879226.361417:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879226.361419:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10165 00000100:00000001:1.0:1788879226.361422:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788879226.361422:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879226.361427:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879226.361428:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879226.361430:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304838185 00000020:00000001:3.0:1788879226.361431:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879226.361434:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879226.361446:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879226.361447:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:3.0:1788879226.361450:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879226.361452:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879226.361453:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879226.361456:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361456:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879226.361461:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788879226.361463:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879226.361467:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879226.361468:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879226/real 1788879226] req@ffff94e50449f480 x1875765290142080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879226.361469:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879226.361473:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879226.361479:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361484:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879226.361486:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000400:00000001:1.0:1788879226.361487:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879226.361489:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00000001:3.0:1788879226.361492:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361494:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879226.361495:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361498:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:3.0:1788879226.361500:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879226.361501:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879226.361504:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361505:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879226.361507:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879226.361508:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879226.361509:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879226.361512:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879226.361514:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290142080/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879226.361517:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361523:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879226.361526:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290142464/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879226.361535:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879226.361536:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879226.361536:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879226.361538:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361542:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361544:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879226.361546:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361548:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879226.361550:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879226.361552:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879226.361556:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879226.361558:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879226.361561:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290142080:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879226.361565:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361566:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879226.361567:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879226.361569:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879226.361572:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879226.361573:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879226.361575:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879226.361577:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879226.361579:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879226.361580:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879226.361582:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361583:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361589:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879226.361590:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361592:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879226.361593:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879226.361594:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879226.361595:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879226.361596:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879226.361597:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879226.361598:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879226.361600:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879226.361601:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290142208:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879226.361614:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879226.361615:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879226.361617:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879226.361618:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879226.361620:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361624:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879226.361624:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:1.0:1788879226.361628:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879226.361635:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879226.361637:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879226.361639:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879226.361640:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361641:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879226.361642:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361645:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304837643 00000100:00000001:1.0:1788879226.361647:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304838185 00000100:00000001:1.0:1788879226.361649:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304837643 00000100:00000001:1.0:1788879226.361650:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304838185 00000100:00000001:1.0:1788879226.361652:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879226.361653:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361654:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879226.361657:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879226/real 1788879226] req@ffff94e50449c700 x1875765290142208/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879226.361662:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879226.361663:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879226.361664:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879226.361665:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879226.361666:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361667:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879226.361668:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879226.361670:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290142208/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879226.361675:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361676:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879226.361677:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361678:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879226.361679:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361680:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879226.361691:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879226.361693:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879226.361695:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879226.361696:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879226.361698:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290142208:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879226.361700:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879226.361701:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879226.361702:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879226.361703:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879226.361704:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879226.361705:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879226.361707:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290142336:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879226.361709:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879226.361710:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879226.361711:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879226.361713:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879226.361714:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361717:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879226.361718:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:1.0:1788879226.361720:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879226.361726:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879226.361728:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879226.361729:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879226.361730:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361731:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879226.361733:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361735:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304837641 00000100:00000001:1.0:1788879226.361736:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304838185 00000100:00000001:1.0:1788879226.361738:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304837641 00000100:00000001:1.0:1788879226.361739:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304838185 00000100:00000001:1.0:1788879226.361742:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879226.361743:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361744:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879226.361746:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879226/real 1788879226] req@ffff94e50449e300 x1875765290142336/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879226.361751:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879226.361752:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879226.361753:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879226.361754:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879226.361755:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361756:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879226.361757:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879226.361759:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290142336/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879226.361764:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361765:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879226.361766:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361767:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879226.361768:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361769:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879226.361777:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879226.361778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879226.361780:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879226.361781:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879226.361783:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290142336:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879226.361785:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879226.361786:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879226.361787:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879226.361788:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879226.361789:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879226.361790:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879226.361792:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290142464:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879226.361794:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879226.361795:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879226.361796:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879226.361798:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879226.361799:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361802:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879226.361803:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:1.0:1788879226.361804:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879226.361809:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879226.361811:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879226.361813:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879226.361813:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361815:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879226.361816:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361818:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304837641 00000100:00000001:1.0:1788879226.361819:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304838185 00000100:00000001:1.0:1788879226.361821:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304837641 00000100:00000001:1.0:1788879226.361823:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304838185 00000100:00000001:1.0:1788879226.361824:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879226.361825:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361826:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879226.361828:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879226/real 1788879226] req@ffff94e50449c000 x1875765290142464/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879226.361833:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879226.361833:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879226.361835:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879226.361835:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879226.361836:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361837:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879226.361838:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879226.361840:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290142464/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879226.361849:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361850:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879226.361852:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361853:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879226.361853:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361854:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879226.361863:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879226.361864:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879226.361866:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879226.361867:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879226.361869:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290142464:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879226.361871:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361873:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879226.361873:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879226.361874:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879226.361876:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879226.361876:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879226.361877:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879226.361879:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879226.361879:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879226.361881:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879226.361882:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361883:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879226.361883:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879226.361884:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879226.361885:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879226.361886:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879226.361887:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879226.361888:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879226.361889:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879226.361890:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879226.361891:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361892:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879226.361892:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879226.361894:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879226.361894:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879226.361896:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879226.361896:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879226.361897:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879226.361898:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879226.361899:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879226.361900:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361922:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879226.361924:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879226.361925:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879226.361926:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879226.361927:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879226.424980:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879226.424985:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879226.424986:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879226.424991:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879226.424993:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879226.424996:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879226.424998:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879226.425000:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879226.425002:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879227.128967:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879227.128971:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879227.128975:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879227.128976:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879227.128992:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879227.128994:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879227.128997:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879227.448983:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879227.448989:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879227.448991:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879227.448996:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879227.448998:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879227.449001:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879227.449003:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879227.449006:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879227.449008:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879228.153048:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879228.153056:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879228.153063:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879228.153065:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879228.153092:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879228.153095:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879228.153099:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879228.473010:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879228.473019:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879228.473022:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879228.473029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879228.473033:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879228.473037:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879228.473041:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879228.473044:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879228.473048:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879229.176976:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879229.176981:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879229.176985:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879229.176986:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879229.177021:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879229.177023:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879229.177025:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879229.497003:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879229.497011:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879229.497013:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879229.497019:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879229.497023:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879229.497027:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879229.497029:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879229.497032:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879229.497035:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879230.201040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879230.201047:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879230.201054:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879230.201056:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879230.201083:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879230.201087:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879230.201090:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879230.521046:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879230.521057:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879230.521060:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879230.521069:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879230.521074:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879230.521080:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879230.521084:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879230.521088:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879230.521092:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879231.225049:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879231.225059:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879231.225068:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879231.225070:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879231.225116:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879231.225121:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879231.225139:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879231.481059:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879231.481074:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879231.481079:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879231.481084:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879231.481091:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879231.481100:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10170 00000100:00080000:3.0:1788879231.481116:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879231.481130:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879231.481137:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879231.481148:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879231.481153:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879231.481159:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879231.481163:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879231.481167:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879231.481170:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879231.481178:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879231.481182:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879231.481187:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879231.481192:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879231.481196:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879231.481200:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879231.481203:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879231.481215:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879231.481217:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879231.481223:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879231.481228:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879231.481234:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879231.481252:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879231.481265:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290142592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879231.481280:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879231.481287:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879231.481339:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879231.481346:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879231.481352:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879231.481354:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879231.481358:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879231.481363:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879231.481368:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10170 00000100:00080000:3.0:1788879231.481373:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879231.481378:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879231.481382:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879231.481387:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879231.481389:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879231.481392:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879231.481395:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879231.481401:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879231.481405:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879231.481406:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879231.481413:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788879231.481415:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879231.481417:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879231.481423:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879231.481423:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.481427:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879231.481430:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879231.481433:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879231.481438:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879231.481444:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879231.481444:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879231.481451:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879231.481453:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290142592:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879231.481455:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879231.481461:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879231.481466:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879231.481469:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879231.481472:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879231.481477:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.481500:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879231.481503:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:2.0:1788879231.481508:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879231.481561:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00100000:3.0:1788879231.481563:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879231.481570:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879231.481574:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.481578:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788879231.481582:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879231.481587:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788879231.481587:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.481591:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879231.481592:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879231.481593:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879231.481597:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879231.481601:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879231.481602:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.481607:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304843305 00000100:00000001:3.0:1788879231.481613:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.481630:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00100000:3.0:1788879231.481633:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290142720/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879231.481640:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304843305 00000100:00000001:2.0:1788879231.481646:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879231.481647:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879231.481652:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.481655:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879231.481658:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.481659:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.481664:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788879231.481666:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788879231.481672:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879231/real 1788879231] req@ffff94e50449ca80 x1875765290142592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879231.481694:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879231.481697:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879231.481701:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879231.481703:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879231.481704:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.481707:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879231.481709:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879231.481712:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290142592/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879231.481718:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.481721:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879231.481724:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.481725:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879231.481725:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.481728:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879231.481729:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879231.481732:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879231.481735:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879231.481737:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879231.481740:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290142592:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879231.481745:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.481746:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879231.481748:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879231.481750:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879231.481753:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879231.481755:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879231.481756:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879231.481760:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879231.481762:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879231.481764:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879231.481765:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.481767:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.481775:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879231.481777:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.481778:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879231.481779:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879231.481780:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879231.481781:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879231.481782:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879231.481784:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879231.481785:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879231.481787:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879231.481789:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290142720:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879231.481792:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879231.481793:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879231.481794:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879231.481795:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879231.481797:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.481802:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879231.481803:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:2.0:1788879231.481807:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879231.481816:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879231.481819:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879231.481821:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879231.481823:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.481824:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879231.481826:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.481830:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304842763 00000100:00000001:2.0:1788879231.481832:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304843305 00000100:00000001:2.0:1788879231.481834:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304842763 00000100:00080000:3.0:1788879231.481834:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879231.481836:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304843305 00000100:00000001:2.0:1788879231.481841:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879231.481841:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879231.481844:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879231.481846:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879231.481847:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879231.481851:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879231/real 1788879231] req@ffff94e50449f100 x1875765290142720/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879231.481855:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000400:00000001:2.0:1788879231.481860:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879231.481864:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1788879231.481864:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10170 00000100:00000001:2.0:1788879231.481867:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879231.481868:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879231.481871:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.481872:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879231.481873:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879231.481876:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290142720/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879231.481883:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.481885:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879231.481886:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.481887:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879231.481888:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.481889:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879231.481947:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879231.481949:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879231.481953:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00080000:2.0:1788879231.481954:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788879231.481955:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879231.481962:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00080000:2.0:1788879231.481963:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000020:00000001:3.0:1788879231.481967:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879231.481970:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:3.0:1788879231.481971:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879231.481977:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00100000:2.0:1788879231.481980:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290142720:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879231.481980:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879231.481985:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879231.481991:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788879231.481992:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879231.481994:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879231.481995:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879231.481998:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879231.481999:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879231.482001:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879231.482004:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00100000:3.0:1788879231.482006:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879231.482009:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000008:00000001:3.0:1788879231.482010:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879231.482014:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879231.482015:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879231.482020:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:3.0:1788879231.482021:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000020:00000001:2.0:1788879231.482022:0:2361:0:(genops.c:1216:class_import_put()) Process entered 02000000:00000001:3.0:1788879231.482024:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879231.482025:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788879231.482029:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879231.482030:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879231.482033:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:3.0:1788879231.482033:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879231.482038:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482040:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879231.482042:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.482043:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879231.482045:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482047:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879231.482050:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482054:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1788879231.482055:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290142848/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879231.482057:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879231.482066:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879231.482068:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879231.482078:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879231.482080:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879231.482084:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879231.482085:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879231.482088:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879231.482091:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879231.482093:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1788879231.482094:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10170 00000100:00000001:2.0:1788879231.482098:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879231.482100:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1788879231.482101:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879231.482104:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879231.482105:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879231.482106:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879231.482108:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879231.482110:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1788879231.482112:0:136860:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1788879231.482114:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788879231.482116:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:2.0:1788879231.482117:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879231.482119:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:2.0:1788879231.482122:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290142848:192.168.203.129@tcp:8:kworker.0 00000020:00000001:3.0:1788879231.482124:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879231.482125:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879231.482129:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788879231.482130:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1788879231.482131:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879231.482134:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879231.482135:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879231.482136:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879231.482144:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879231.482146:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879231.482150:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:3.0:1788879231.482150:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879231.482157:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879231.482158:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788879231.482164:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879231.482168:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788879231.482172:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879231.482172:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879231.482177:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879231.482179:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00000001:3.0:1788879231.482181:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482183:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879231.482184:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482198:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482200:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879231.482202:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879231.482203:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879231.482205:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482207:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304842762 00000100:00000001:3.0:1788879231.482208:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879231.482211:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879231.482212:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304843305 02000000:00000001:3.0:1788879231.482214:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482216:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304842762 00000100:00000001:2.0:1788879231.482219:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304843305 00000100:00000001:2.0:1788879231.482222:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879231.482222:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482227:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482229:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879231.482232:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290142976/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788879231.482235:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879231/real 1788879231] req@ffff94e50449ca80 x1875765290142848/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879231.482256:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:2.0:1788879231.482258:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879231.482261:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879231.482261:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482266:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879231.482268:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482269:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879231.482271:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.482273:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879231.482274:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879231.482278:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290142848/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879231.482283:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.482285:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879231.482287:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.482288:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879231.482289:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.482290:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879231.482319:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879231.482321:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879231.482323:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879231.482337:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879231.482340:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290142848:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879231.482342:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.482344:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879231.482345:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879231.482346:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879231.482348:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879231.482349:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879231.482350:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879231.482351:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879231.482353:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879231.482354:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879231.482355:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.482356:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.482358:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879231.482359:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482361:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879231.482362:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879231.482363:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879231.482364:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879231.482365:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879231.482367:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879231.482368:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879231.482370:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879231.482372:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290142976:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879231.482374:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879231.482375:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879231.482376:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879231.482377:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879231.482379:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482382:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879231.482383:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:2.0:1788879231.482386:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879231.482393:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879231.482396:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879231.482397:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879231.482399:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482400:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879231.482402:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482404:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304842762 00000100:00000001:2.0:1788879231.482406:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304843305 00000100:00000001:2.0:1788879231.482408:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304842762 00000100:00000001:2.0:1788879231.482410:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304843305 00000100:00000001:2.0:1788879231.482411:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879231.482412:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482413:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879231.482416:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879231/real 1788879231] req@ffff94e50449c000 x1875765290142976/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879231.482420:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879231.482421:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879231.482423:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879231.482424:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879231.482425:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.482426:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879231.482427:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879231.482430:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290142976/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879231.482433:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.482435:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879231.482436:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.482437:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879231.482438:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.482439:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879231.482451:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879231.482453:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879231.482455:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879231.482456:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879231.482459:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290142976:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879231.482462:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.482463:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879231.482464:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879231.482465:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879231.482466:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879231.482467:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879231.482467:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879231.482469:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879231.482469:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879231.482470:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879231.482471:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.482472:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879231.482474:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879231.482475:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879231.482477:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879231.482478:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879231.545016:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879231.545026:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879231.545029:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879231.545047:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879231.545053:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879231.545059:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879231.545062:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879231.545066:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879231.545069:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879232.249038:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879232.249045:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879232.249050:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879232.249051:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879232.249074:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879232.249076:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879232.249079:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879232.569048:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879232.569056:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879232.569059:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879232.569065:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879232.569069:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879232.569073:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879232.569075:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879232.569078:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879232.569081:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879233.273078:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879233.273086:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879233.273091:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879233.273095:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879233.273097:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879233.273101:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879233.273103:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879233.273105:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879233.273107:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879233.273109:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879233.273214:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879233.273217:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879233.273220:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879233.593030:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879233.593040:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879233.593042:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879233.593048:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879233.593052:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879233.593057:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879233.593060:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879233.593063:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879233.593066:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879234.297089:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879234.297100:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879234.297109:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879234.297111:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879234.297159:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879234.297163:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879234.297168:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879234.617147:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879234.617169:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879234.617173:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879234.617181:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879234.617186:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879234.617192:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879234.617196:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879234.617200:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879234.617204:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879235.321150:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879235.321169:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879235.321175:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879235.321184:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879235.321195:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879235.321200:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879235.321203:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879235.321207:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879235.321213:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879235.321215:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879235.321312:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879235.321316:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879235.321320:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879235.641117:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879235.641133:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879235.641138:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879235.641150:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879235.641159:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879235.641168:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879235.641173:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879235.641178:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879235.641184:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879236.345047:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879236.345082:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879236.345096:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879236.345099:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879236.345147:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879236.345154:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879236.345161:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879236.601035:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879236.601050:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879236.601055:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879236.601060:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879236.601067:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879236.601077:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10175 00000100:00080000:3.0:1788879236.601091:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879236.601095:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879236.601098:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879236.601108:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879236.601111:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879236.601115:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879236.601119:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879236.601121:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879236.601123:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879236.601129:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879236.601132:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879236.601136:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879236.601141:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879236.601144:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879236.601147:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879236.601150:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879236.601164:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879236.601166:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879236.601170:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879236.601173:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879236.601179:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879236.601194:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879236.601206:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290143104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879236.601220:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879236.601226:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879236.601249:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879236.601254:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879236.601280:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879236.601283:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879236.601285:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879236.601289:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879236.601294:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10175 00000100:00080000:3.0:1788879236.601301:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879236.601304:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879236.601306:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879236.601311:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879236.601315:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879236.601318:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879236.601321:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879236.601324:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879236.601328:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879236.601333:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879236.601336:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879236.601339:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879236.601347:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879236.601350:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879236.601352:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879236.601358:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:3.0:1788879236.601358:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879236.601373:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.601375:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879236.601379:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.601382:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788879236.601388:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879236.601389:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788879236.601397:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.601398:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879236.601413:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879236.601416:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879236.601418:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879236.601421:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879236.601426:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879236.601432:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879236.601434:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879236.601449:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290143232/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788879236.601456:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290143104:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879236.601483:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879236.601494:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788879236.601496:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788879236.601503:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879236.601504:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879236.601529:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879236.601534:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879236.601540:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1788879236.601544:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879236.601547:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879236.601554:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879236.601557:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879236.601562:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879236.601567:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10175 00000100:00080000:3.0:1788879236.601573:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879236.601575:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879236.601577:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879236.601580:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879236.601582:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:2.0:1788879236.601599:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1788879236.601603:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879236.601609:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879236.601613:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879236.601621:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879236.601626:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879236.601630:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879236.601639:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879236.601646:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879236.601668:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:2.0:1788879236.601671:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879236.601679:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879236.601689:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879236.601693:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879236.601697:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00100000:3.0:1788879236.601699:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879236.601706:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.601709:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879236.601710:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.601717:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.601723:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879236.601723:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879236.601736:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.601741:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879236.601743:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879236.601757:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879236.601761:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304848425 02000000:00000001:3.0:1788879236.601764:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879236.601779:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.601784:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00100000:3.0:1788879236.601792:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290143360/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879236.601800:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304848425 00000100:00000001:3.0:1788879236.601820:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879236.601821:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879236.601828:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.601832:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879236.601835:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.601838:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879236.601843:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879236.601845:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788879236.601857:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879236.601866:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1788879236.601868:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879236/real 1788879236] req@ffff94e50449e300 x1875765290143104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879236.601878:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879236.601954:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879236.601970:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10175 00000400:00000001:2.0:1788879236.601975:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879236.601983:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1788879236.601985:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879236.601993:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879236.601997:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879236.601998:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879236.602014:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879236.602015:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000020:00000001:3.0:1788879236.602018:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879236.602027:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788879236.602028:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879236.602046:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879236.602047:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879236.602053:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879236.602056:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879236.602060:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879236.602067:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879236.602080:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879236.602084:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1788879236.602092:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290143104/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879236.602095:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879236.602115:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879236.602124:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879236.602128:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879236.602133:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879236.602145:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879236.602158:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879236.602161:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879236.602162:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879236.602178:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879236.602181:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879236.602186:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788879236.602186:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.602210:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879236.602214:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.602218:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879236.602223:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879236.602235:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290143488/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879236.602237:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879236.602247:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879236.602287:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788879236.602298:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879236.602305:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879236.602308:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879236.602312:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.602323:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879236.602339:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290143104:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879236.602358:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.602364:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879236.602367:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879236.602372:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879236.602379:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879236.602383:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879236.602386:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879236.602392:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879236.602395:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879236.602399:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879236.602401:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.602404:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.602415:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879236.602418:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.602422:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879236.602424:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879236.602427:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879236.602429:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879236.602431:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879236.602436:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879236.602439:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879236.602442:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879236.602447:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290143232:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879236.602452:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879236.602455:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879236.602458:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879236.602460:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879236.602464:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.602474:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879236.602476:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:2.0:1788879236.602483:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879236.602505:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879236.602512:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879236.602515:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879236.602517:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.602520:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879236.602524:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.602530:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304847884 00000100:00000001:2.0:1788879236.602534:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304848425 00000100:00000001:2.0:1788879236.602537:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304847884 00000100:00000001:2.0:1788879236.602540:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304848425 00000100:00000001:2.0:1788879236.602542:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879236.602545:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.602547:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879236.602553:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879236/real 1788879236] req@ffff94e50449c700 x1875765290143232/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879236.602564:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879236.602566:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879236.602569:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879236.602571:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879236.602573:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.602576:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879236.602578:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879236.602583:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290143232/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879236.602592:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.602595:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879236.602598:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.602600:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879236.602601:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.602604:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879236.602625:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879236.602628:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879236.602632:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879236.602635:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879236.602640:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290143232:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879236.602645:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879236.602647:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879236.602649:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879236.602653:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879236.602655:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879236.602657:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879236.602661:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290143360:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879236.602664:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879236.602667:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879236.602669:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879236.602671:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879236.602675:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.602681:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879236.602683:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:2.0:1788879236.602688:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879236.602700:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879236.602705:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879236.602708:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879236.602709:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.602712:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879236.602715:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.602719:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304847882 00000100:00000001:2.0:1788879236.602723:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304848425 00000100:00000001:2.0:1788879236.602725:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304847882 00000100:00000001:2.0:1788879236.602728:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304848425 00000100:00000001:2.0:1788879236.602731:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879236.602733:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.602735:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879236.602740:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879236/real 1788879236] req@ffff94e50449f480 x1875765290143360/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879236.602759:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879236.602760:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879236.602763:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879236.602765:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879236.602767:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.602769:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879236.602770:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879236.602775:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290143360/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879236.602783:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.602785:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879236.602788:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.602790:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879236.602792:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.602794:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879236.602813:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879236.602816:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879236.602820:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879236.602823:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879236.602827:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290143360:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879236.602832:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879236.602834:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879236.602835:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879236.602838:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879236.602840:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879236.602843:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879236.602846:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290143488:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879236.602850:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879236.602852:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879236.602855:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879236.602857:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879236.602860:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.602865:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879236.602867:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:2.0:1788879236.602872:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879236.602882:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879236.602887:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879236.602889:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879236.602891:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.602893:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879236.602896:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.602900:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304847882 00000100:00000001:2.0:1788879236.602947:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304848425 00000100:00000001:2.0:1788879236.602952:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304847883 00000100:00000001:2.0:1788879236.602956:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304848425 00000100:00000001:2.0:1788879236.602960:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879236.602964:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.602968:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879236.602977:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879236/real 1788879236] req@ffff94e50449f800 x1875765290143488/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879236.602995:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879236.602998:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879236.603005:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879236.603009:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879236.603012:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.603015:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879236.603018:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879236.603025:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290143488/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879236.603034:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.603037:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879236.603041:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.603043:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879236.603045:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.603047:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879236.603066:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879236.603070:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879236.603074:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879236.603077:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879236.603081:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290143488:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879236.603085:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.603088:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879236.603090:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879236.603092:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879236.603095:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879236.603097:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879236.603099:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879236.603103:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879236.603106:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879236.603110:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879236.603113:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.603116:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879236.603118:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879236.603121:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879236.603124:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879236.603126:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879236.603128:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879236.603132:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879236.603134:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879236.603137:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879236.603140:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.603143:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879236.603146:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879236.603148:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879236.603153:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879236.603156:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879236.603158:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879236.603162:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879236.603165:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879236.603168:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879236.603170:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.603174:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879236.603181:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879236.603184:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879236.603188:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879236.603191:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879236.665070:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879236.665078:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879236.665080:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879236.665088:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879236.665092:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879236.665097:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879236.665100:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879236.665104:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879236.665107:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879237.369104:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879237.369115:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879237.369123:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879237.369125:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879237.369160:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879237.369164:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879237.369168:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879237.689054:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879237.689065:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879237.689068:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879237.689075:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879237.689081:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879237.689087:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879237.689091:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879237.689095:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879237.689099:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879238.393043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879238.393054:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879238.393064:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879238.393067:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879238.393100:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879238.393104:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879238.393107:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879238.713055:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879238.713066:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879238.713069:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879238.713076:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879238.713082:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879238.713087:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879238.713091:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879238.713095:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879238.713099:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879239.417049:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879239.417056:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879239.417062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879239.417064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879239.417110:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879239.417114:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879239.417128:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879239.736997:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879239.737004:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879239.737007:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879239.737012:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879239.737016:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879239.737020:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879239.737022:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879239.737025:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879239.737027:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879240.441000:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879240.441007:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879240.441012:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879240.441014:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879240.441033:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879240.441036:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879240.441038:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879240.761062:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879240.761071:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879240.761074:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879240.761096:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879240.761100:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879240.761104:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879240.761117:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879240.761120:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879240.761123:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879241.465043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879241.465051:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879241.465058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879241.465061:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879241.465095:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879241.465111:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879241.465118:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879241.721103:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879241.721116:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879241.721119:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879241.721123:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879241.721128:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879241.721136:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10180 00000100:00080000:3.0:1788879241.721149:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879241.721152:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879241.721171:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879241.721180:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879241.721182:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879241.721186:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879241.721188:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879241.721190:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879241.721192:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879241.721196:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879241.721198:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879241.721201:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879241.721205:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879241.721207:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879241.721210:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879241.721213:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879241.721241:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879241.721244:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879241.721248:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879241.721252:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879241.721258:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879241.721281:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879241.721291:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290143616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879241.721302:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879241.721307:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879241.721324:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879241.721338:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879241.721342:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879241.721343:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879241.721345:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879241.721348:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879241.721351:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10180 00000100:00080000:3.0:1788879241.721355:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879241.721357:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879241.721359:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879241.721363:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879241.721364:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879241.721367:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879241.721370:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879241.721372:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879241.721374:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879241.721378:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879241.721382:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.721383:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788879241.721386:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879241.721390:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879241.721395:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788879241.721396:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879241.721398:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879241.721402:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879241.721405:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879241.721409:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879241.721412:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:2.0:1788879241.721414:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879241.721420:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788879241.721421:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879241.721427:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290143616:192.168.203.129@tcp:250:kworker.0 00000008:00000001:3.0:1788879241.721431:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.721436:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879241.721438:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879241.721442:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879241.721447:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879241.721449:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879241.721450:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879241.721456:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879241.721460:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879241.721463:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879241.721469:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879241.721473:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.721481:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879241.721485:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.721486:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:2.0:1788879241.721495:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1788879241.721501:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290143744/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879241.721511:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879241.721514:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879241.721518:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879241.721521:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879241.721525:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879241.721527:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879241.721529:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879241.721532:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879241.721534:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10180 00000100:00080000:3.0:1788879241.721539:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1788879241.721542:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879241.721543:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879241.721549:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879241.721554:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879241.721557:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879241.721560:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879241.721561:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:3.0:1788879241.721567:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879241.721570:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879241.721571:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879241.721577:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879241.721578:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.721583:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:3.0:1788879241.721587:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879241.721592:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.721593:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879241.721595:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879241.721600:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304853545 02000000:00000001:3.0:1788879241.721601:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879241.721609:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879241.721611:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879241.721615:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879241.721617:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00000001:3.0:1788879241.721618:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879241.721626:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.721631:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304853545 02000000:00000001:3.0:1788879241.721632:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879241.721636:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879241.721639:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879241.721641:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879241.721643:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879241.721649:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.721654:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00100000:3.0:1788879241.721655:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290143872/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879241.721667:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879241.721669:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879241.721670:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879241.721679:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.721680:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879241.721683:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879241.721688:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788879241.721689:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879241.721696:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879241.721698:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879241.721709:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879241.721734:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10180 00000100:00100000:2.0:1788879241.721735:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879241/real 1788879241] req@ffff94e50449d880 x1875765290143616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879241.721740:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879241.721747:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879241.721751:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879241.721759:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879241.721763:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879241.721768:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879241.721776:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879241.721783:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879241.721785:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:2.0:1788879241.721787:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1788879241.721792:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000400:00000001:2.0:1788879241.721795:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879241.721798:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879241.721803:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879241.721807:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879241.721826:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879241.721828:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879241.721830:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879241.721832:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.721833:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879241.721834:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879241.721844:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879241.721845:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:3.0:1788879241.721847:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.721851:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879241.721854:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879241.721859:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788879241.721861:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879241.721865:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879241.721868:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879241.721896:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290143616/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879241.721897:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879241.721917:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290144000/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879241.721933:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879241.721938:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879241.721944:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.721975:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.721982:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879241.721987:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.721990:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879241.721994:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722014:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879241.722018:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879241.722023:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879241.722041:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879241.722047:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879241.722054:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290143616:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879241.722061:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722064:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879241.722067:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879241.722071:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879241.722076:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879241.722079:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879241.722083:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879241.722089:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879241.722092:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879241.722096:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879241.722098:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722101:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722111:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879241.722113:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.722133:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879241.722135:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879241.722137:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879241.722139:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879241.722142:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879241.722145:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879241.722147:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879241.722150:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879241.722155:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290143744:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879241.722161:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879241.722164:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879241.722167:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879241.722169:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879241.722172:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.722180:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879241.722181:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:2.0:1788879241.722186:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879241.722202:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879241.722207:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879241.722210:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879241.722211:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.722213:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879241.722216:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.722220:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304853004 00000100:00000001:2.0:1788879241.722243:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304853545 00000100:00000001:2.0:1788879241.722246:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304853004 00000100:00000001:2.0:1788879241.722248:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304853545 00000100:00000001:2.0:1788879241.722250:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879241.722252:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.722255:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879241.722260:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879241/real 1788879241] req@ffff94e50449fb80 x1875765290143744/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879241.722277:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879241.722278:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879241.722282:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879241.722283:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879241.722285:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722287:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879241.722288:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879241.722291:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290143744/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879241.722298:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722300:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879241.722302:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722304:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879241.722305:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722307:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879241.722320:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879241.722323:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879241.722336:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879241.722338:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879241.722341:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290143744:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879241.722344:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879241.722346:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879241.722347:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879241.722349:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879241.722351:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879241.722353:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879241.722356:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290143872:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879241.722361:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879241.722363:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879241.722365:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879241.722367:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879241.722370:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.722375:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879241.722376:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:2.0:1788879241.722380:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879241.722392:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879241.722397:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879241.722399:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879241.722400:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.722402:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879241.722404:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.722408:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304853002 00000100:00000001:2.0:1788879241.722410:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304853545 00000100:00000001:2.0:1788879241.722412:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304853002 00000100:00000001:2.0:1788879241.722414:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304853545 00000100:00000001:2.0:1788879241.722416:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879241.722418:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.722420:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879241.722423:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879241/real 1788879241] req@ffff94e50449c380 x1875765290143872/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879241.722430:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879241.722431:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879241.722434:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879241.722435:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879241.722436:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722438:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879241.722440:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879241.722443:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290143872/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879241.722455:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722458:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879241.722460:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722461:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879241.722462:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722464:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879241.722476:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879241.722478:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879241.722480:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879241.722482:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879241.722485:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290143872:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879241.722488:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879241.722489:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879241.722491:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879241.722493:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879241.722495:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879241.722496:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879241.722499:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290144000:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879241.722503:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879241.722505:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879241.722507:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879241.722510:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879241.722514:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.722522:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879241.722524:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:2.0:1788879241.722529:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879241.722545:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879241.722552:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879241.722555:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879241.722557:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.722560:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879241.722564:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.722569:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304853002 00000100:00000001:2.0:1788879241.722573:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304853545 00000100:00000001:2.0:1788879241.722577:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304853002 00000100:00000001:2.0:1788879241.722580:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304853545 00000100:00000001:2.0:1788879241.722584:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879241.722587:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.722589:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879241.722595:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879241/real 1788879241] req@ffff94e50449d180 x1875765290144000/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879241.722604:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879241.722605:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879241.722608:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879241.722610:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879241.722611:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722613:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879241.722614:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879241.722618:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290144000/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879241.722624:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722627:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879241.722629:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722630:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879241.722631:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722633:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879241.722645:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879241.722647:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879241.722650:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879241.722651:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879241.722655:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290144000:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879241.722658:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722660:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879241.722662:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879241.722663:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879241.722665:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879241.722667:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879241.722668:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879241.722671:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879241.722672:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879241.722674:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879241.722676:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722677:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879241.722678:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879241.722680:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879241.722682:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879241.722684:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879241.722686:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879241.722689:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879241.722691:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879241.722693:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879241.722695:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722698:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879241.722699:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879241.722701:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879241.722703:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879241.722715:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879241.722733:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879241.722736:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879241.722738:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879241.722741:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879241.722743:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722747:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879241.722754:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879241.722756:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879241.722760:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879241.722763:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879241.784986:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879241.784993:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879241.784994:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879241.784999:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879241.785002:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879241.785005:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879241.785008:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879241.785010:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879241.785012:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879242.489054:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879242.489062:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879242.489069:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879242.489070:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879242.489094:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879242.489097:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879242.489101:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879242.809028:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879242.809037:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879242.809041:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879242.809050:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879242.809057:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879242.809063:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879242.809067:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879242.809072:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879242.809077:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879243.513004:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879243.513011:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879243.513016:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879243.513019:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879243.513032:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879243.513036:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879243.513038:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879243.513041:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879243.513043:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879243.513045:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879243.513067:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879243.513070:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879243.513073:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879243.833000:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879243.833006:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879243.833008:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879243.833011:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879243.833036:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879243.833039:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879243.833042:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879243.833044:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879243.833046:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879244.537001:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879244.537006:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879244.537009:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879244.537010:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879244.537026:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879244.537028:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879244.537031:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879244.857043:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879244.857051:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879244.857054:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879244.857061:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879244.857067:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879244.857073:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879244.857077:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879244.857082:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879244.857087:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879245.561051:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879245.561075:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879245.561080:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879245.561086:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879245.561093:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879245.561097:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879245.561101:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879245.561106:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879245.561113:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879245.561116:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879245.561152:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879245.561157:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879245.561162:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879245.881003:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879245.881009:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879245.881012:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879245.881017:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879245.881021:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879245.881025:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879245.881027:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879245.881030:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879245.881033:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879246.585006:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879246.585011:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879246.585015:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879246.585016:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879246.585033:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879246.585035:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879246.585037:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879246.840994:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879246.841001:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879246.841003:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879246.841007:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879246.841010:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879246.841016:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10185 00000100:00080000:3.0:1788879246.841025:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879246.841027:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879246.841028:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879246.841035:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879246.841037:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879246.841040:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879246.841041:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879246.841043:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879246.841044:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879246.841048:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879246.841050:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879246.841052:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879246.841055:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879246.841057:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879246.841058:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879246.841060:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879246.841070:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879246.841071:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879246.841073:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879246.841076:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879246.841079:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879246.841089:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879246.841097:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290144128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879246.841105:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879246.841109:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879246.841122:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879246.841124:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879246.841127:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879246.841128:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879246.841129:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879246.841132:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879246.841133:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10185 00000100:00080000:3.0:1788879246.841135:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879246.841136:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879246.841138:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879246.841140:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879246.841141:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879246.841142:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879246.841144:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879246.841145:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879246.841148:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879246.841150:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879246.841152:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879246.841153:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879246.841158:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879246.841159:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879246.841161:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879246.841163:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:3.0:1788879246.841164:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879246.841168:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00000001:3.0:1788879246.841168:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841171:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879246.841172:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841176:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879246.841179:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879246.841179:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879246.841183:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879246.841183:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879246.841196:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879246.841199:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879246.841200:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879246.841203:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879246.841205:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879246.841209:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290144128:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879246.841214:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841215:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879246.841219:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879246.841222:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879246.841225:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1788879246.841225:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290144256/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788879246.841229:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879246.841239:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879246.841242:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841246:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879246.841246:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841249:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000100:00080000:3.0:1788879246.841250:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879246.841253:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879246.841255:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879246.841257:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879246.841259:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879246.841261:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879246.841263:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10185 00000100:00080000:3.0:1788879246.841274:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879246.841276:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879246.841277:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879246.841280:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879246.841282:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879246.841283:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000800:00000100:2.0:1788879246.841287:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:3.0:1788879246.841288:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879246.841292:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879246.841294:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879246.841299:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:3.0:1788879246.841300:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879246.841303:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879246.841305:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841306:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841310:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879246.841310:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879246.841316:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879246.841317:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879246.841320:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304858665 02000000:00000001:3.0:1788879246.841321:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879246.841326:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879246.841328:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879246.841329:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00000001:3.0:1788879246.841334:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841337:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304858665 00000100:00000001:3.0:1788879246.841337:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841344:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1788879246.841344:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879246.841347:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841348:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879246.841349:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879246.841351:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879246.841352:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879246.841356:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1788879246.841356:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879246.841363:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879246/real 1788879246] req@ffff94e50449e680 x1875765290144128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879246.841364:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879246.841373:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290144384/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1788879246.841377:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879246.841379:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879246.841383:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879246.841385:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879246.841388:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841390:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:3.0:1788879246.841390:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879246.841394:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00080000:3.0:1788879246.841395:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879246.841396:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879246.841397:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879246.841400:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1788879246.841401:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879246.841404:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879246.841405:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879246.841410:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10185 00000100:00100000:2.0:1788879246.841411:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290144128/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879246.841414:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879246.841419:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879246.841421:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879246.841424:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788879246.841425:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879246.841428:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1788879246.841428:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879246.841432:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788879246.841432:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879246.841437:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879246.841438:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879246.841440:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879246.841442:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879246.841443:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879246.841444:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879246.841446:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879246.841450:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00080000:2.0:1788879246.841451:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788879246.841453:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879246.841455:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:2.0:1788879246.841458:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 02000000:00000001:3.0:1788879246.841459:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879246.841461:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1788879246.841462:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879246.841466:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290144128:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879246.841466:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879246.841468:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879246.841472:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841473:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879246.841475:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841476:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879246.841479:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879246.841481:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879246.841482:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879246.841483:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879246.841485:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1788879246.841486:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:2.0:1788879246.841490:0:2361:0:(genops.c:1216:class_import_put()) Process entered 02000000:00000001:3.0:1788879246.841490:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:2.0:1788879246.841494:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879246.841494:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879246.841499:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1788879246.841500:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879246.841501:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879246.841505:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879246.841508:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879246.841508:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290144512/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879246.841513:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879246.841519:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879246.841521:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841522:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879246.841525:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879246.841526:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841529:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879246.841531:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879246.841532:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879246.841534:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879246.841535:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879246.841537:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879246.841539:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879246.841541:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879246.841545:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290144256:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879246.841548:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879246.841550:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879246.841551:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879246.841552:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879246.841555:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841559:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879246.841560:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:2.0:1788879246.841564:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879246.841572:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879246.841575:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879246.841576:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879246.841577:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841578:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879246.841580:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841583:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304858123 00000100:00000001:2.0:1788879246.841585:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304858665 00000100:00000001:2.0:1788879246.841587:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304858123 00000100:00000001:2.0:1788879246.841588:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304858665 00000100:00000001:2.0:1788879246.841590:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879246.841591:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841592:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879246.841595:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879246/real 1788879246] req@ffff94e50449ce00 x1875765290144256/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879246.841600:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879246.841601:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879246.841602:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879246.841603:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879246.841604:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841605:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879246.841606:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879246.841609:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290144256/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879246.841614:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841615:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879246.841617:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841618:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879246.841619:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841620:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879246.841628:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879246.841630:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879246.841632:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879246.841633:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879246.841635:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290144256:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879246.841637:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879246.841638:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879246.841639:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879246.841641:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879246.841642:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879246.841643:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879246.841645:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290144384:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879246.841647:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879246.841648:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879246.841649:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879246.841650:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879246.841652:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841655:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879246.841656:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:2.0:1788879246.841659:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879246.841664:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879246.841667:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879246.841668:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879246.841669:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841670:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879246.841672:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841674:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304858121 00000100:00000001:2.0:1788879246.841675:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304858665 00000100:00000001:2.0:1788879246.841677:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304858121 00000100:00000001:2.0:1788879246.841678:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304858665 00000100:00000001:2.0:1788879246.841680:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879246.841681:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841682:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879246.841684:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879246/real 1788879246] req@ffff94e50449df80 x1875765290144384/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879246.841689:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879246.841690:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879246.841691:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879246.841692:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879246.841692:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841693:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879246.841694:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879246.841697:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290144384/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879246.841701:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841703:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879246.841704:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841705:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879246.841706:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841707:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879246.841714:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879246.841715:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879246.841717:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879246.841718:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879246.841721:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290144384:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879246.841723:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879246.841724:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879246.841725:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879246.841726:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879246.841727:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879246.841729:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879246.841730:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290144512:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879246.841732:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879246.841733:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879246.841734:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879246.841735:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879246.841736:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841739:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879246.841739:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:2.0:1788879246.841742:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879246.841747:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879246.841750:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879246.841751:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879246.841752:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841753:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879246.841754:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841757:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304858121 00000100:00000001:2.0:1788879246.841758:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304858665 00000100:00000001:2.0:1788879246.841760:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304858121 00000100:00000001:2.0:1788879246.841761:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304858665 00000100:00000001:2.0:1788879246.841763:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879246.841764:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841765:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879246.841767:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879246/real 1788879246] req@ffff94e50449d180 x1875765290144512/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879246.841776:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879246.841777:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879246.841779:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879246.841780:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879246.841781:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841782:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879246.841782:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879246.841800:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290144512/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879246.841806:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841808:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879246.841810:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841811:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879246.841813:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841814:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879246.841825:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879246.841827:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879246.841830:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879246.841832:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879246.841835:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290144512:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879246.841846:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841848:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879246.841849:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879246.841850:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879246.841853:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879246.841854:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879246.841855:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879246.841857:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879246.841859:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879246.841861:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879246.841862:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841864:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879246.841865:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879246.841867:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879246.841868:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879246.841885:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879246.841886:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879246.841887:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879246.841888:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879246.841889:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879246.841890:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841891:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879246.841892:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879246.841892:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879246.841894:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879246.841894:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879246.841895:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879246.841896:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879246.841897:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879246.841897:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879246.841898:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841899:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879246.841901:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879246.841902:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879246.841903:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879246.841904:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879246.904965:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879246.904970:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879246.904972:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879246.904976:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879246.904979:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879246.904982:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879246.904984:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879246.904986:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879246.904988:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879247.608995:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879247.609001:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879247.609005:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879247.609007:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879247.609027:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879247.609030:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879247.609032:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879247.929005:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879247.929014:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879247.929017:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879247.929023:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879247.929027:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879247.929032:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879247.929035:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879247.929039:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879247.929042:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879248.633021:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879248.633029:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879248.633034:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879248.633036:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879248.633062:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879248.633067:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879248.633072:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879248.952998:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879248.953005:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879248.953007:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879248.953012:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879248.953015:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879248.953019:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879248.953022:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879248.953025:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879248.953027:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879249.656988:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879249.657012:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879249.657019:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879249.657021:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879249.657050:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879249.657055:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879249.657060:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879249.977000:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879249.977008:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879249.977009:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879249.977013:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879249.977016:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879249.977019:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879249.977021:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879249.977023:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879249.977025:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879250.681008:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879250.681017:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879250.681025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879250.681027:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879250.681052:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879250.681055:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879250.681059:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879251.000973:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879251.000978:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879251.000998:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879251.001005:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879251.001009:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879251.001014:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879251.001017:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879251.001021:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879251.001025:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879251.705026:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879251.705033:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879251.705040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879251.705042:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879251.705067:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879251.705070:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879251.705073:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879251.961072:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879251.961087:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879251.961092:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879251.961098:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879251.961106:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879251.961115:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10190 00000100:00080000:2.0:1788879251.961127:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879251.961131:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879251.961134:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879251.961144:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879251.961148:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879251.961181:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879251.961188:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879251.961194:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879251.961196:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879251.961205:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879251.961211:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879251.961216:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879251.961221:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879251.961227:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879251.961231:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879251.961235:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879251.961250:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879251.961253:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879251.961259:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879251.961263:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879251.961271:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879251.961289:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879251.961305:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290144640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879251.961326:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879251.961334:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879251.961363:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879251.961370:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879251.961375:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879251.961377:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879251.961379:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879251.961384:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879251.961388:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10190 00000100:00080000:2.0:1788879251.961396:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879251.961400:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879251.961402:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879251.961408:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879251.961410:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879251.961413:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879251.961418:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788879251.961418:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879251.961427:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879251.961428:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879251.961434:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879251.961436:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879251.961443:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788879251.961446:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879251.961448:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879251.961455:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879251.961459:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879251.961461:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879251.961466:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879251.961471:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879251.961472:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879251.961484:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879251.961492:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:1.0:1788879251.961493:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290144640:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1788879251.961501:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879251.961504:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1788879251.961509:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879251.961512:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788879251.961515:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879251.961518:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879251.961523:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879251.961524:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879251.961529:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879251.961531:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879251.961533:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879251.961541:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879251.961546:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879251.961557:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.961559:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879251.961564:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00100000:2.0:1788879251.961568:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290144768/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879251.961574:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879251.961587:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879251.961591:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879251.961596:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879251.961600:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879251.961604:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879251.961607:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879251.961609:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879251.961612:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879251.961616:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10190 00000100:00080000:2.0:1788879251.961622:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879251.961624:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879251.961627:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879251.961630:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879251.961633:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879251.961635:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000800:00000100:1.0:1788879251.961645:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:2.0:1788879251.961646:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879251.961655:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879251.961657:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879251.961664:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879251.961668:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.961671:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:2.0:1788879251.961672:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879251.961680:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879251.961684:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788879251.961685:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.961691:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879251.961692:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879251.961699:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879251.961700:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000008:00000001:2.0:1788879251.961708:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879251.961711:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.961713:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879251.961717:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879251.961721:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.961723:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304863785 00000100:00000001:2.0:1788879251.961724:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879251.961729:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879251.961741:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1788879251.961748:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879251.961758:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879251.961767:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290144896/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879251.961772:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304863785 00000100:00000001:1.0:1788879251.961786:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879251.961787:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879251.961794:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.961799:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879251.961800:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879251.961808:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879251.961812:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879251.961814:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879251.961820:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879251.961824:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1788879251.961825:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879251.961835:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879251.961844:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10190 00000100:00100000:1.0:1788879251.961850:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879251/real 1788879251] req@ffff94e5045e7480 x1875765290144640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879251.961850:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879251.961857:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879251.961861:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879251.961870:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879251.961873:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879251.961878:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879251.961887:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879251.961892:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879251.961894:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:1.0:1788879251.961895:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1788879251.961942:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879251.961948:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879251.961950:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:1.0:1788879251.961954:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1788879251.961955:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879251.961963:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.961966:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879251.961967:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879251.961976:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879251.961977:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879251.961983:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1788879251.961985:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.961989:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879251.961991:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.961994:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879251.962000:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879251.962006:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879251.962009:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290144640/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879251.962011:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879251.962017:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879251.962023:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.962035:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879251.962036:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.962043:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:2.0:1788879251.962052:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290145024/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879251.962053:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962060:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879251.962064:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962072:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879251.962078:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879251.962080:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00080000:1.0:1788879251.962085:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788879251.962087:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879251.962098:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879251.962102:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879251.962109:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879251.962127:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290144640:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879251.962135:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962138:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879251.962140:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879251.962143:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879251.962150:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879251.962180:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879251.962183:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879251.962189:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879251.962191:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879251.962195:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879251.962197:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962200:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962209:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879251.962211:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.962214:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879251.962216:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879251.962218:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879251.962220:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879251.962222:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879251.962226:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879251.962228:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879251.962231:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879251.962236:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290144768:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879251.962241:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879251.962244:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879251.962246:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879251.962248:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879251.962253:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.962262:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879251.962264:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:1.0:1788879251.962269:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879251.962289:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879251.962296:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879251.962299:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879251.962300:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.962303:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879251.962307:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.962312:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304863244 00000100:00000001:1.0:1788879251.962316:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304863785 00000100:00000001:1.0:1788879251.962319:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304863244 00000100:00000001:1.0:1788879251.962322:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304863785 00000100:00000001:1.0:1788879251.962325:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879251.962328:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.962330:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879251.962336:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879251/real 1788879251] req@ffff94e5045e4700 x1875765290144768/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879251.962347:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879251.962349:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879251.962352:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879251.962354:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879251.962355:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962358:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879251.962359:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879251.962364:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290144768/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879251.962373:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962376:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879251.962379:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962380:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879251.962382:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962384:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879251.962398:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879251.962401:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879251.962406:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879251.962408:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879251.962413:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290144768:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879251.962418:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879251.962419:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879251.962421:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879251.962424:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879251.962426:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879251.962428:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879251.962432:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290144896:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879251.962436:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879251.962438:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879251.962440:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879251.962442:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879251.962446:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.962451:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879251.962453:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:1.0:1788879251.962457:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879251.962469:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879251.962473:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879251.962476:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879251.962477:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.962479:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879251.962482:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.962486:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304863242 00000100:00000001:1.0:1788879251.962489:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304863785 00000100:00000001:1.0:1788879251.962492:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304863242 00000100:00000001:1.0:1788879251.962495:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304863785 00000100:00000001:1.0:1788879251.962497:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879251.962499:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.962501:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879251.962506:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879251/real 1788879251] req@ffff94e5045e6680 x1875765290144896/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879251.962516:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879251.962517:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879251.962520:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879251.962522:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879251.962523:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962526:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879251.962528:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879251.962532:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290144896/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879251.962549:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962552:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879251.962555:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962556:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879251.962558:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962560:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879251.962573:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879251.962578:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879251.962585:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879251.962589:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879251.962595:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290144896:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879251.962602:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879251.962604:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879251.962607:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879251.962611:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879251.962614:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879251.962618:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879251.962625:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290145024:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879251.962632:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879251.962636:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879251.962639:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879251.962643:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879251.962649:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.962659:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879251.962662:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:1.0:1788879251.962668:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879251.962689:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879251.962695:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879251.962698:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879251.962700:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.962703:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879251.962706:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.962711:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304863242 00000100:00000001:1.0:1788879251.962714:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304863785 00000100:00000001:1.0:1788879251.962717:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304863242 00000100:00000001:1.0:1788879251.962719:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304863785 00000100:00000001:1.0:1788879251.962722:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879251.962724:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.962727:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879251.962732:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879251/real 1788879251] req@ffff94e5045e6a00 x1875765290145024/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879251.962741:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879251.962743:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879251.962746:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879251.962748:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879251.962750:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962752:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879251.962754:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879251.962758:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290145024/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879251.962767:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962770:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879251.962773:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962774:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879251.962776:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962778:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879251.962793:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879251.962798:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879251.962804:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879251.962807:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879251.962815:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290145024:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879251.962823:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962826:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879251.962828:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879251.962832:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879251.962836:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879251.962839:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879251.962841:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879251.962846:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879251.962849:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879251.962852:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879251.962855:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962859:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879251.962861:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879251.962864:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879251.962868:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879251.962870:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879251.962872:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879251.962875:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879251.962877:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879251.962881:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879251.962884:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962887:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879251.962888:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879251.962890:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879251.962892:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879251.962893:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879251.962895:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879251.962897:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879251.962898:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879251.962900:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879251.962930:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962933:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879251.962938:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879251.962940:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879251.962942:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879251.962944:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879252.025015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879252.025021:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879252.025024:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879252.025029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879252.025033:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879252.025037:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879252.025039:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879252.025042:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879252.025045:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879252.729040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879252.729048:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879252.729054:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879252.729056:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879252.729082:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879252.729086:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879252.729089:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879253.049009:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879253.049018:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879253.049022:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879253.049029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879253.049035:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879253.049041:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879253.049045:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879253.049049:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879253.049054:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879253.753025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879253.753033:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879253.753039:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879253.753043:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879253.753046:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879253.753050:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879253.753053:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879253.753056:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879253.753058:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879253.753061:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879253.753129:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879253.753135:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879253.753168:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879254.073018:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879254.073029:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879254.073033:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879254.073041:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879254.073048:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879254.073053:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879254.073058:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879254.073063:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879254.073086:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879254.777051:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879254.777060:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879254.777067:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879254.777069:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879254.777127:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879254.777143:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879254.777148:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879255.096986:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879255.096995:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879255.096999:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879255.097009:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879255.097030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879255.097034:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879255.097038:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879255.097041:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879255.097045:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879255.801006:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879255.801015:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879255.801018:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879255.801022:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879255.801027:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879255.801041:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879255.801044:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879255.801047:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879255.801051:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879255.801053:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879255.801088:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879255.801091:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879255.801095:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879256.121010:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879256.121020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879256.121023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879256.121032:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879256.121039:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879256.121045:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879256.121049:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879256.121055:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879256.121060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879256.824980:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879256.824985:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879256.824989:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879256.824990:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879256.825023:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879256.825025:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879256.825027:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879257.080949:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879257.080956:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879257.080959:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879257.080963:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879257.080967:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879257.080974:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10195 00000100:00080000:2.0:1788879257.080985:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879257.080988:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879257.080990:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879257.080998:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879257.081001:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879257.081005:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879257.081008:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879257.081010:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879257.081011:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879257.081015:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879257.081017:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879257.081020:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879257.081022:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879257.081025:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879257.081026:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879257.081028:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879257.081035:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879257.081036:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879257.081038:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879257.081041:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879257.081044:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879257.081053:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879257.081060:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290145152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879257.081068:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879257.081072:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879257.081083:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879257.081086:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879257.081090:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879257.081092:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879257.081093:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879257.081095:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879257.081097:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10195 00000100:00080000:2.0:1788879257.081101:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879257.081102:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879257.081103:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879257.081106:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879257.081107:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879257.081109:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788879257.081109:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879257.081114:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788879257.081114:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879257.081127:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879257.081128:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879257.081130:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879257.081132:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788879257.081136:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879257.081137:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879257.081141:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879257.081143:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879257.081144:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879257.081147:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879257.081151:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879257.081151:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879257.081157:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879257.081160:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:1.0:1788879257.081161:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290145152:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1788879257.081164:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879257.081167:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1788879257.081168:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879257.081170:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879257.081173:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879257.081175:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879257.081177:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879257.081178:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879257.081178:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879257.081182:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879257.081183:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879257.081188:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879257.081191:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879257.081193:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879257.081201:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290145280/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879257.081209:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879257.081212:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:1.0:1788879257.081218:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879257.081224:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879257.081228:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879257.081232:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879257.081235:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879257.081239:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879257.081241:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879257.081243:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879257.081246:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000800:00000100:1.0:1788879257.081251:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788879257.081251:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10195 00000100:00080000:2.0:1788879257.081273:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879257.081276:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879257.081278:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879257.081280:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:2.0:1788879257.081282:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879257.081284:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879257.081286:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1788879257.081287:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879257.081290:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879257.081291:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879257.081295:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879257.081296:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879257.081300:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879257.081303:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879257.081303:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879257.081306:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879257.081308:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879257.081309:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304868905 02000000:00000001:2.0:1788879257.081311:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879257.081314:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879257.081316:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879257.081318:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879257.081319:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00000001:2.0:1788879257.081320:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879257.081324:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081325:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304868905 02000000:00000001:2.0:1788879257.081329:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879257.081331:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1788879257.081333:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081335:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879257.081338:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879257.081339:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879257.081343:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879257.081344:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1788879257.081348:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879257.081353:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879257/real 1788879257] req@ffff94e5045e5880 x1875765290145152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879257.081358:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879257.081367:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879257.081370:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:2.0:1788879257.081372:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290145408/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879257.081374:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879257.081377:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879257.081380:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.081385:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879257.081388:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879257.081389:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879257.081393:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879257.081396:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879257.081399:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290145152/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879257.081399:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879257.081406:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879257.081409:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879257.081411:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879257.081417:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879257.081421:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10195 00000100:00000001:1.0:1788879257.081425:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879257.081427:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879257.081430:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879257.081432:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879257.081433:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1788879257.081435:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879257.081437:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879257.081441:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879257.081442:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788879257.081443:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879257.081448:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879257.081448:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879257.081452:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879257.081452:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879257.081458:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1788879257.081459:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879257.081478:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879257.081479:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879257.081483:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:1.0:1788879257.081484:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1788879257.081486:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879257.081489:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879257.081490:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879257.081492:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879257.081494:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00100000:2.0:1788879257.081495:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879257.081500:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290145152:192.168.203.129@tcp:250:kworker.0 00000008:00000001:2.0:1788879257.081500:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879257.081504:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081510:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879257.081511:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879257.081514:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788879257.081514:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081517:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788879257.081518:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879257.081521:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879257.081522:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879257.081541:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:2.0:1788879257.081541:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879257.081545:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879257.081549:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:2.0:1788879257.081550:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879257.081553:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879257.081557:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00100000:2.0:1788879257.081559:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290145536/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879257.081562:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879257.081566:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.081571:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879257.081578:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879257.081582:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081583:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879257.081585:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081587:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081593:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879257.081595:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879257.081596:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879257.081598:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879257.081599:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879257.081601:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879257.081604:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879257.081606:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879257.081609:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290145280:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879257.081612:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879257.081613:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879257.081615:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879257.081616:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879257.081618:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081624:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879257.081625:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 00000100:00000001:1.0:1788879257.081628:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879257.081657:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879257.081664:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879257.081667:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879257.081669:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081671:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879257.081675:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081681:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304868363 00000100:00000001:1.0:1788879257.081686:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304868905 00000100:00000001:1.0:1788879257.081690:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304868363 00000100:00000001:1.0:1788879257.081694:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304868905 00000100:00000001:1.0:1788879257.081698:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879257.081702:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081706:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879257.081714:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879257/real 1788879257] req@ffff94e5045e5180 x1875765290145280/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879257.081744:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879257.081745:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879257.081748:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879257.081759:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879257.081760:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.081763:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879257.081764:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879257.081769:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290145280/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879257.081775:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.081777:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879257.081779:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.081780:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879257.081781:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.081782:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879257.081792:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879257.081793:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879257.081795:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879257.081797:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879257.081799:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290145280:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879257.081802:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879257.081803:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879257.081804:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879257.081806:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879257.081807:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879257.081809:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879257.081811:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290145408:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879257.081813:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879257.081814:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879257.081815:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879257.081816:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879257.081819:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081823:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879257.081824:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:1.0:1788879257.081827:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879257.081836:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879257.081839:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879257.081841:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879257.081842:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081844:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879257.081845:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081848:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304868361 00000100:00000001:1.0:1788879257.081849:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304868905 00000100:00000001:1.0:1788879257.081851:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304868361 00000100:00000001:1.0:1788879257.081853:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304868905 00000100:00000001:1.0:1788879257.081854:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879257.081855:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081856:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879257.081859:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879257/real 1788879257] req@ffff94e5045e4a80 x1875765290145408/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879257.081864:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879257.081865:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879257.081866:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879257.081867:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879257.081868:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.081869:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879257.081870:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879257.081873:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290145408/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879257.081877:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.081878:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879257.081880:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.081881:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879257.081882:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.081883:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879257.081891:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879257.081892:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879257.081894:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879257.081895:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879257.081898:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290145408:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879257.081919:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879257.081921:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879257.081922:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879257.081923:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879257.081925:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879257.081926:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879257.081928:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290145536:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879257.081930:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879257.081932:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879257.081933:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879257.081934:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879257.081936:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081940:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879257.081941:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:1.0:1788879257.081943:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879257.081950:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879257.081953:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879257.081955:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879257.081956:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081957:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879257.081959:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081961:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304868362 00000100:00000001:1.0:1788879257.081963:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304868905 00000100:00000001:1.0:1788879257.081964:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304868362 00000100:00000001:1.0:1788879257.081966:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304868905 00000100:00000001:1.0:1788879257.081967:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879257.081968:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.081969:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879257.081972:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879257/real 1788879257] req@ffff94e5045e7100 x1875765290145536/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879257.081989:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879257.081991:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879257.081993:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879257.081995:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879257.081996:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.081998:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879257.082000:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879257.082004:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290145536/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879257.082011:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.082013:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879257.082016:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.082017:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879257.082019:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.082020:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879257.082034:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879257.082037:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879257.082041:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879257.082042:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879257.082045:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290145536:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879257.082047:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.082049:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879257.082050:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879257.082051:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879257.082053:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879257.082054:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879257.082055:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879257.082057:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879257.082058:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879257.082059:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879257.082060:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.082062:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879257.082063:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879257.082063:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879257.082065:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879257.082066:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879257.082067:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879257.082068:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879257.082068:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879257.082069:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879257.082070:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.082071:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879257.082072:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879257.082072:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879257.082073:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879257.082074:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879257.082075:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879257.082076:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879257.082077:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879257.082078:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879257.082079:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.082080:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879257.082082:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879257.082083:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879257.082084:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879257.082085:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879257.144964:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879257.144968:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879257.144970:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879257.144973:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879257.144976:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879257.144978:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879257.144980:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879257.144982:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879257.144984:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879257.849036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879257.849045:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879257.849051:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879257.849053:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879257.849096:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879257.849101:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879257.849108:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879258.168997:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879258.169007:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879258.169010:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879258.169017:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879258.169022:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879258.169027:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879258.169030:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879258.169033:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879258.169037:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879258.873025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879258.873033:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879258.873040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879258.873041:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879258.873099:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879258.873121:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879258.873127:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879259.192997:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879259.193005:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879259.193008:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879259.193015:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879259.193019:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879259.193024:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879259.193028:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879259.193032:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879259.193035:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879259.897085:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879259.897135:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879259.897145:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879259.897147:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879259.897195:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879259.897200:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879259.897205:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879260.217008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879260.217016:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879260.217019:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879260.217025:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879260.217030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879260.217034:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879260.217038:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879260.217041:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879260.217045:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879260.921066:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879260.921084:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879260.921114:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879260.921118:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879260.921207:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879260.921213:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879260.921219:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879261.241081:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879261.241122:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879261.241126:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879261.241135:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879261.241142:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879261.241149:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879261.241154:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879261.241159:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879261.241164:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879261.945103:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879261.945114:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879261.945122:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879261.945125:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879261.945168:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879261.945172:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879261.945176:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879262.201001:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879262.201010:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879262.201013:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879262.201016:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879262.201021:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879262.201027:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10201 00000100:00080000:2.0:1788879262.201037:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879262.201040:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879262.201042:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879262.201050:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879262.201052:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879262.201056:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879262.201058:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879262.201060:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879262.201061:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879262.201065:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879262.201067:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879262.201070:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879262.201073:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879262.201076:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201078:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879262.201094:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879262.201106:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879262.201108:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201110:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879262.201113:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879262.201117:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201129:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879262.201138:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290145664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879262.201147:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879262.201151:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201166:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879262.201169:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879262.201175:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879262.201177:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879262.201178:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879262.201181:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879262.201183:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10201 00000100:00080000:2.0:1788879262.201186:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879262.201188:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879262.201189:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879262.201192:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879262.201193:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879262.201195:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879262.201197:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879262.201199:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879262.201201:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879262.201204:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879262.201207:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879262.201209:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879262.201215:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879262.201216:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879262.201221:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.201223:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879262.201223:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879262.201228:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:2.0:1788879262.201230:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879262.201232:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:2.0:1788879262.201235:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.201236:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201239:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879262.201243:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879262.201245:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879262.201248:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879262.201248:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201254:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879262.201256:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879262.201257:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879262.201262:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879262.201267:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290145664:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879262.201269:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.201277:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:2.0:1788879262.201281:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290145792/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879262.201283:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879262.201287:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879262.201293:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879262.201298:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879262.201300:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201301:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201310:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879262.201312:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879262.201315:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879262.201316:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879262.201317:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879262.201319:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879262.201321:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10201 00000100:00000001:1.0:1788879262.201322:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788879262.201324:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879262.201327:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879262.201328:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879262.201330:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879262.201331:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879262.201332:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879262.201334:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879262.201337:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:2.0:1788879262.201338:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879262.201341:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879262.201345:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788879262.201348:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879262.201351:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879262.201353:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879262.201355:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879262.201357:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201358:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879262.201360:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879262.201362:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201364:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879262.201368:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879262.201369:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201370:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879262.201372:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879262.201374:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201378:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879262.201382:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290145920/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788879262.201388:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879262.201397:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879262.201400:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201403:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.201404:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788879262.201407:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879262.201410:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788879262.201412:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879262.201415:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201416:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879262.201420:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879262.201421:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879262.201423:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879262.201429:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879262.201429:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10201 00000100:00000001:1.0:1788879262.201436:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304874025 00000100:00080000:2.0:1788879262.201437:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879262.201442:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879262.201445:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879262.201447:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879262.201450:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879262.201451:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1788879262.201452:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879262.201458:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879262.201460:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879262.201461:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879262.201462:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304874025 00000100:00080000:2.0:1788879262.201466:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879262.201470:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879262.201471:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879262.201475:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879262.201477:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879262.201479:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879262.201481:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879262.201483:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.201486:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879262.201487:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879262.201492:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879262.201494:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201498:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879262.201499:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879262/real 1788879262] req@ffff94e5045e5c00 x1875765290145664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879262.201502:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879262.201505:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201510:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879262.201512:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879262.201515:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879262.201519:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788879262.201521:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879262.201522:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879262.201528:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00100000:2.0:1788879262.201531:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290146048/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879262.201532:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879262.201536:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.201541:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879262.201548:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879262.201550:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879262.201552:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.201553:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879262.201562:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290145664/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879262.201573:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.201577:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879262.201581:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.201584:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879262.201585:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.201588:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879262.201590:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879262.201595:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879262.201601:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879262.201605:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879262.201610:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290145664:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879262.201616:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.201619:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879262.201621:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879262.201624:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879262.201628:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879262.201631:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879262.201634:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879262.201638:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879262.201639:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879262.201642:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879262.201643:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.201645:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.201652:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879262.201654:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.201656:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879262.201657:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879262.201659:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879262.201660:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879262.201662:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879262.201664:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879262.201666:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879262.201667:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879262.201670:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290145792:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879262.201673:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879262.201675:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879262.201677:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879262.201678:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879262.201681:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.201688:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879262.201689:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:1.0:1788879262.201693:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879262.201706:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879262.201710:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879262.201720:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879262.201722:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.201723:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879262.201726:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.201730:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304873483 00000100:00000001:1.0:1788879262.201733:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304874025 00000100:00000001:1.0:1788879262.201735:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304873483 00000100:00000001:1.0:1788879262.201737:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304874025 00000100:00000001:1.0:1788879262.201739:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879262.201740:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.201742:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879262.201745:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879262/real 1788879262] req@ffff94e5045e5500 x1875765290145792/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879262.201752:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879262.201753:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879262.201755:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879262.201756:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879262.201758:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.201759:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879262.201760:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879262.201764:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290145792/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879262.201770:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.201771:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879262.201773:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.201774:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879262.201775:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.201777:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879262.201788:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879262.201790:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879262.201793:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879262.201795:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879262.201798:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290145792:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879262.201801:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879262.201802:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879262.201803:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879262.201805:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879262.201806:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879262.201808:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879262.201810:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290145920:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879262.201813:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879262.201814:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879262.201815:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879262.201817:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879262.201819:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.201823:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879262.201824:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 00000100:00000001:1.0:1788879262.201827:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879262.201835:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879262.201839:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879262.201848:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879262.201849:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.201850:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879262.201853:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.201855:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304873481 00000100:00000001:1.0:1788879262.201858:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304874025 00000100:00000001:1.0:1788879262.201859:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304873481 00000100:00000001:1.0:1788879262.201862:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304874025 00000100:00000001:1.0:1788879262.201863:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879262.201865:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.201866:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879262.201870:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879262/real 1788879262] req@ffff94e5045e6300 x1875765290145920/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879262.201877:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879262.201878:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879262.201880:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879262.201882:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879262.201883:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.201884:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879262.201885:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879262.201888:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290145920/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879262.201894:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.201895:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879262.201897:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.201899:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879262.201900:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.201931:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879262.201940:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879262.201943:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879262.201946:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879262.201948:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879262.201951:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290145920:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879262.201960:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879262.201961:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879262.201963:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879262.201965:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879262.201966:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879262.201968:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879262.201970:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290146048:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879262.201973:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879262.201975:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879262.201976:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879262.201977:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879262.201980:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.201984:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879262.201985:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000100:00000001:1.0:1788879262.201988:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879262.201998:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879262.202001:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879262.202003:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879262.202004:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.202005:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879262.202008:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.202010:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304873482 00000100:00000001:1.0:1788879262.202013:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304874025 00000100:00000001:1.0:1788879262.202014:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304873482 00000100:00000001:1.0:1788879262.202016:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304874025 00000100:00000001:1.0:1788879262.202018:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879262.202019:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.202021:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879262.202024:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879262/real 1788879262] req@ffff94e5045e7800 x1875765290146048/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879262.202030:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879262.202031:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879262.202033:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879262.202034:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879262.202035:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.202036:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879262.202038:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879262.202041:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290146048/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879262.202046:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.202048:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879262.202050:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.202051:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879262.202052:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.202054:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879262.202104:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879262.202108:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879262.202113:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879262.202117:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879262.202122:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290146048:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879262.202128:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.202131:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879262.202133:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879262.202136:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879262.202139:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879262.202141:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879262.202143:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879262.202145:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879262.202147:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879262.202150:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879262.202152:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.202154:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879262.202155:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879262.202157:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879262.202159:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879262.202161:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879262.202164:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879262.202167:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879262.202169:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879262.202172:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879262.202175:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.202177:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879262.202179:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879262.202180:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879262.202183:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879262.202184:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879262.202185:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879262.202188:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879262.202190:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879262.202192:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879262.202194:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.202196:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879262.202201:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879262.202203:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879262.202206:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879262.202208:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879262.264976:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879262.264981:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879262.264982:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879262.264986:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879262.264989:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879262.264991:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879262.264993:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879262.265012:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879262.265015:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879262.969024:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879262.969031:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879262.969037:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879262.969038:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879262.969061:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879262.969064:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879262.969067:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879263.288984:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879263.288991:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879263.288993:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879263.288998:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879263.289002:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879263.289006:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879263.289008:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879263.289011:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879263.289014:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879263.993014:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879263.993024:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879263.993033:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879263.993038:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879263.993041:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879263.993045:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879263.993049:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879263.993052:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879263.993054:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879263.993057:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879263.993102:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879263.993105:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879263.993108:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879264.312988:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879264.312994:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879264.312996:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879264.313001:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879264.313004:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879264.313007:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879264.313009:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879264.313011:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879264.313013:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879265.017042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879265.017050:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879265.017059:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879265.017079:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879265.017107:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879265.017110:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879265.017113:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879265.337048:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879265.337057:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879265.337080:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879265.337088:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879265.337094:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879265.337099:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879265.337103:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879265.337107:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879265.337110:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879266.040999:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879266.041007:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879266.041009:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879266.041012:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879266.041016:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879266.041018:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879266.041020:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879266.041022:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879266.041025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879266.041026:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879266.041041:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879266.041044:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879266.041046:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879266.361028:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879266.361037:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879266.361040:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879266.361049:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879266.361080:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879266.361086:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879266.361107:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879266.361111:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879266.361114:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879267.065070:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879267.065079:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879267.065102:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879267.065104:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879267.065120:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879267.065122:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879267.065125:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879267.321104:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879267.321117:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879267.321121:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879267.321126:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879267.321132:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879267.321141:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10206 00000100:00080000:0.0:1788879267.321173:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879267.321178:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879267.321182:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879267.321194:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879267.321199:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879267.321205:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879267.321221:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879267.321226:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879267.321229:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879267.321239:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879267.321243:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879267.321246:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879267.321250:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879267.321254:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321257:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879267.321260:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879267.321290:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879267.321292:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321296:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879267.321298:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879267.321303:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321315:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879267.321326:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290146176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879267.321342:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879267.321367:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321392:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879267.321399:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879267.321405:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879267.321408:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879267.321412:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879267.321418:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879267.321425:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10206 00000100:00080000:0.0:1788879267.321434:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879267.321439:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879267.321441:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879267.321442:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879267.321452:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879267.321452:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788879267.321459:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879267.321460:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788879267.321464:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879267.321465:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879267.321471:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879267.321473:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:3.0:1788879267.321479:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788879267.321481:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788879267.321484:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879267.321487:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:3.0:1788879267.321489:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879267.321503:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:3.0:1788879267.321505:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290146176:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879267.321515:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.321517:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879267.321524:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879267.321525:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879267.321537:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879267.321543:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321546:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879267.321553:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879267.321558:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321562:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879267.321569:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879267.321571:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321574:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879267.321576:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879267.321580:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321589:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879267.321600:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979180 x1875765290146304/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879267.321614:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788879267.321615:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321618:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321623:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879267.321626:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879267.321629:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879267.321631:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879267.321633:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879267.321636:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879267.321656:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10206 00000100:00080000:0.0:1788879267.321660:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879267.321662:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879267.321663:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879267.321666:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879267.321668:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879267.321671:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879267.321673:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879267.321675:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879267.321676:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879267.321679:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879267.321681:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879267.321683:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879267.321685:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879267.321691:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879267.321693:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879267.321694:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879267.321699:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879267.321701:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321703:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879267.321704:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879267.321709:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879267.321712:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321714:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879267.321716:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879267.321718:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321722:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879267.321727:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290146432/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879267.321734:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879267.321736:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321738:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879267.321741:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879267.321744:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879267.321745:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879267.321746:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879267.321749:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879267.321752:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10206 00000100:00080000:0.0:1788879267.321755:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879267.321757:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879267.321759:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879267.321762:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879267.321763:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879267.321765:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879267.321767:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879267.321769:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879267.321770:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879267.321773:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879267.321775:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879267.321776:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879267.321778:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879267.321780:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321781:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879267.321783:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879267.321785:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321786:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879267.321791:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879267.321792:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321794:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879267.321795:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879267.321797:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321819:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879267.321824:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290146560/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879267.321832:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879267.321834:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.321837:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.321880:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879267.321884:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000100:00000001:3.0:1788879267.321890:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879267.322032:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879267.322075:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879267.322080:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879267.322082:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322084:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879267.322089:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322094:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304879146 00000100:00000001:3.0:1788879267.322104:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879267.322110:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304879146 00000100:00000001:3.0:1788879267.322113:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879267.322117:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879267.322120:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322122:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879267.322129:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879267/real 1788879267] req@ffff94e50897b800 x1875765290146176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879267.322140:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879267.322142:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879267.322146:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879267.322165:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879267.322167:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322170:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879267.322172:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879267.322178:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290146176/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879267.322187:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322191:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879267.322194:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322196:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879267.322197:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322200:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879267.322202:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879267.322205:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879267.322209:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879267.322212:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879267.322216:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290146176:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879267.322221:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322224:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879267.322225:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879267.322228:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879267.322232:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879267.322234:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879267.322236:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879267.322240:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879267.322242:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879267.322246:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879267.322247:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322250:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322257:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879267.322260:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322262:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879267.322263:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879267.322265:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879267.322267:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879267.322291:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879267.322293:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879267.322295:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879267.322298:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879267.322301:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290146304:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879267.322305:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879267.322307:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879267.322309:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879267.322310:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879267.322313:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322319:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879267.322320:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 00000100:00000001:3.0:1788879267.322325:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879267.322338:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879267.322359:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879267.322362:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879267.322365:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322367:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879267.322372:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322378:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304878604 00000100:00000001:3.0:1788879267.322382:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304879146 00000100:00000001:3.0:1788879267.322387:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304878604 00000100:00000001:3.0:1788879267.322390:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304879146 00000100:00000001:3.0:1788879267.322394:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879267.322397:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322401:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879267.322408:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879267/real 1788879267] req@ffff94e508979180 x1875765290146304/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879267.322424:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879267.322426:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879267.322431:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879267.322434:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879267.322436:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322439:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879267.322442:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879267.322449:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290146304/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879267.322468:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322471:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879267.322474:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322476:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879267.322477:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322479:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879267.322491:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879267.322494:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879267.322497:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879267.322500:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879267.322503:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290146304:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879267.322508:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879267.322509:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879267.322511:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879267.322513:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879267.322516:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879267.322518:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879267.322522:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290146432:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879267.322525:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879267.322528:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879267.322529:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879267.322531:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879267.322534:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322542:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879267.322543:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 00000100:00000001:3.0:1788879267.322548:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879267.322562:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879267.322568:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879267.322570:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879267.322572:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322574:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879267.322577:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322581:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304878602 00000100:00000001:3.0:1788879267.322584:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304879146 00000100:00000001:3.0:1788879267.322586:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304878602 00000100:00000001:3.0:1788879267.322589:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304879146 00000100:00000001:3.0:1788879267.322591:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879267.322593:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322595:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879267.322599:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879267/real 1788879267] req@ffff94e508978380 x1875765290146432/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879267.322607:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879267.322608:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879267.322610:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879267.322612:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879267.322613:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322615:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879267.322616:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879267.322620:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290146432/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879267.322628:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322630:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879267.322633:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322635:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879267.322636:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322638:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879267.322670:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879267.322675:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879267.322679:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879267.322682:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879267.322688:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290146432:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879267.322694:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879267.322695:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879267.322698:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879267.322701:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879267.322704:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879267.322707:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879267.322712:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290146560:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879267.322717:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879267.322720:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879267.322722:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879267.322724:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879267.322728:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322736:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879267.322737:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:3.0:1788879267.322741:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879267.322753:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879267.322758:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879267.322760:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879267.322762:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322763:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879267.322766:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322769:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304878602 00000100:00000001:3.0:1788879267.322772:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304879146 00000100:00000001:3.0:1788879267.322774:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304878602 00000100:00000001:3.0:1788879267.322777:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304879146 00000100:00000001:3.0:1788879267.322779:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879267.322780:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322782:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879267.322788:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879267/real 1788879267] req@ffff94e50897bb80 x1875765290146560/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879267.322808:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879267.322826:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879267.322830:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879267.322833:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879267.322835:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322838:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879267.322841:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879267.322848:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290146560/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879267.322861:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322865:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879267.322869:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322873:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879267.322875:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322878:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879267.322891:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879267.322894:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879267.322897:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879267.322900:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879267.322904:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290146560:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879267.322908:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322910:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879267.322912:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879267.322914:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879267.322917:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879267.322918:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879267.322920:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879267.322923:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879267.322925:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879267.322928:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879267.322946:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322949:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879267.322950:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879267.322952:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879267.322953:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879267.322955:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879267.322956:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879267.322958:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879267.322959:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879267.322960:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879267.322961:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322963:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879267.322964:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879267.322965:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879267.322967:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879267.322969:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879267.322971:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879267.322974:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879267.322975:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879267.322978:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879267.322980:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322982:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879267.322985:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879267.322987:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879267.322990:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879267.322991:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879267.384995:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879267.385000:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879267.385002:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879267.385007:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879267.385010:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879267.385014:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879267.385018:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879267.385021:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879267.385024:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879268.089123:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879268.089131:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879268.089137:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879268.089139:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879268.089161:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879268.089164:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879268.089168:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879268.409016:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879268.409025:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879268.409027:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879268.409033:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879268.409055:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879268.409060:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879268.409063:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879268.409067:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879268.409070:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879269.112992:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879269.112997:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879269.113001:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879269.113002:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879269.113017:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879269.113020:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879269.113024:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879269.433060:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879269.433066:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879269.433068:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879269.433072:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879269.433087:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879269.433090:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879269.433093:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879269.433095:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879269.433097:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879270.137002:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879270.137012:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879270.137022:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879270.137043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879270.137091:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879270.137094:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879270.137098:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879270.457010:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879270.457018:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879270.457021:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879270.457053:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879270.457067:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879270.457072:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879270.457076:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879270.457079:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879270.457082:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879271.161114:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879271.161122:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879271.161129:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879271.161131:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879271.161154:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879271.161157:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879271.161161:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879271.481053:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879271.481082:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879271.481086:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879271.481094:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879271.481100:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879271.481106:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879271.481110:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879271.481114:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879271.481119:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879272.185036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879272.185045:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879272.185052:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879272.185054:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879272.185077:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879272.185080:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879272.185083:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879272.441027:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879272.441043:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879272.441047:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879272.441052:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879272.441058:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879272.441067:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10211 00000100:00080000:0.0:1788879272.441080:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879272.441084:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879272.441087:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879272.441098:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879272.441101:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879272.441105:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879272.441109:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879272.441111:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879272.441113:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879272.441119:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879272.441122:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879272.441127:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879272.441132:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879272.441138:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879272.441143:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879272.441147:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879272.441164:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879272.441168:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879272.441173:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879272.441178:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879272.441187:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879272.441206:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879272.441224:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290146688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879272.441241:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879272.441247:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879272.441273:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879272.441281:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879272.441287:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879272.441290:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879272.441294:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879272.441299:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879272.441304:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10211 00000100:00080000:0.0:1788879272.441314:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879272.441317:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879272.441320:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879272.441327:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879272.441330:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879272.441334:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879272.441339:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879272.441343:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879272.441346:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879272.441350:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1788879272.441353:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879272.441356:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879272.441357:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879272.441362:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879272.441362:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879272.441366:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879272.441369:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879272.441371:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879272.441375:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879272.441376:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788879272.441379:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788879272.441380:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879272.441385:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788879272.441386:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879272.441392:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879272.441394:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290146688:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879272.441396:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.441402:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879272.441405:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879272.441407:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879272.441409:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788879272.441411:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879272.441413:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879272.441414:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879272.441418:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879272.441428:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879272.441433:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1788879272.441438:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290146816/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879272.441440:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.441462:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879272.441464:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879272.441466:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:0.0:1788879272.441469:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.441473:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879272.441474:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879272.441479:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879272.441484:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879272.441487:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879272.441490:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879272.441494:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879272.441499:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10211 00000100:00080000:0.0:1788879272.441517:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879272.441522:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879272.441525:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:3.0:1788879272.441527:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:0.0:1788879272.441531:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879272.441534:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879272.441537:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879272.441541:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879272.441545:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879272.441546:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879272.441548:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879272.441553:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1788879272.441555:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879272.441557:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879272.441559:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.441561:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:0.0:1788879272.441563:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879272.441569:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879272.441570:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879272.441573:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879272.441576:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879272.441577:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304884265 00000008:00100000:0.0:1788879272.441580:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879272.441585:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879272.441588:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.441591:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1788879272.441594:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879272.441597:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.441600:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304884265 00000100:00000001:0.0:1788879272.441601:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879272.441604:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879272.441608:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.441608:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788879272.441617:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.441626:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788879272.441628:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290146944/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879272.441632:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.441636:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879272.441643:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:3.0:1788879272.441646:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879272/real 1788879272] req@ffff94e508978e00 x1875765290146688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879272.441648:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879272.441652:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879272.441657:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:3.0:1788879272.441666:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879272.441670:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:0.0:1788879272.441677:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879272.441677:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879272.441682:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879272.441685:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879272.441687:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879272.441690:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879272.441690:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879272.441694:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879272.441696:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879272.441701:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10211 00000100:00100000:3.0:1788879272.441703:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290146688/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879272.441708:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879272.441714:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879272.441717:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879272.441718:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788879272.441726:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879272.441726:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1788879272.441731:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879272.441734:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879272.441734:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.441737:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1788879272.441741:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879272.441741:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879272.441745:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879272.441746:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879272.441748:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879272.441750:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879272.441755:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00080000:3.0:1788879272.441756:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1788879272.441760:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879272.441764:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:3.0:1788879272.441764:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879272.441768:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:0.0:1788879272.441769:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879272.441773:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879272.441776:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290146688:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879272.441777:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879272.441781:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879272.441785:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788879272.441787:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.441789:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879272.441790:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.441792:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1788879272.441797:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879272.441797:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1788879272.441800:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879272.441802:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1788879272.441803:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:3.0:1788879272.441805:0:2361:0:(genops.c:1216:class_import_put()) Process entered 02000000:00000001:0.0:1788879272.441806:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:3.0:1788879272.441809:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879272.441810:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879272.441815:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1788879272.441819:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879272.441819:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879272.441824:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879272.441827:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879272.441828:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290147072/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879272.441832:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879272.441841:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879272.441842:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879272.441846:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.441846:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879272.441851:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.441851:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879272.441853:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879272.441856:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879272.441859:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879272.441861:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879272.441866:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879272.441869:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879272.441873:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879272.441879:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290146816:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879272.441887:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879272.441891:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879272.441894:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879272.441897:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879272.441946:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.441958:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879272.441960:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:3.0:1788879272.441965:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879272.441987:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879272.441993:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879272.441997:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879272.441999:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.442001:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879272.442005:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.442039:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304883724 00000100:00000001:3.0:1788879272.442043:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304884265 00000100:00000001:3.0:1788879272.442047:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304883724 00000100:00000001:3.0:1788879272.442049:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304884265 00000100:00000001:3.0:1788879272.442052:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879272.442055:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.442058:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879272.442063:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879272/real 1788879272] req@ffff94e50897ad80 x1875765290146816/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879272.442074:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879272.442076:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879272.442079:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879272.442082:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879272.442084:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442086:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879272.442088:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879272.442093:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290146816/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879272.442102:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442105:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879272.442108:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442110:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879272.442111:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442113:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879272.442126:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879272.442139:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879272.442144:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879272.442147:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879272.442151:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290146816:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879272.442156:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879272.442158:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879272.442159:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879272.442162:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879272.442164:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879272.442167:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879272.442171:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290146944:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879272.442175:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879272.442177:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879272.442179:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879272.442181:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879272.442186:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.442192:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879272.442194:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:3.0:1788879272.442199:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879272.442213:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879272.442218:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879272.442221:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879272.442223:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.442225:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879272.442229:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.442233:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304883722 00000100:00000001:3.0:1788879272.442236:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304884265 00000100:00000001:3.0:1788879272.442238:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304883722 00000100:00000001:3.0:1788879272.442242:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304884265 00000100:00000001:3.0:1788879272.442245:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879272.442247:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.442249:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879272.442255:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879272/real 1788879272] req@ffff94e508979c00 x1875765290146944/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879272.442265:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879272.442267:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879272.442270:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879272.442272:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879272.442274:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442276:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879272.442278:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879272.442282:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290146944/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879272.442290:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442293:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879272.442295:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442297:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879272.442299:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442301:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879272.442312:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879272.442315:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879272.442319:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879272.442322:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879272.442326:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290146944:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879272.442330:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879272.442332:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879272.442333:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879272.442336:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879272.442338:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879272.442340:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879272.442343:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290147072:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879272.442347:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879272.442350:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879272.442352:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879272.442354:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879272.442357:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.442362:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879272.442364:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:3.0:1788879272.442368:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879272.442379:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879272.442383:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879272.442386:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879272.442387:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.442390:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879272.442393:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.442396:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304883722 00000100:00000001:3.0:1788879272.442399:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304884265 00000100:00000001:3.0:1788879272.442402:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304883722 00000100:00000001:3.0:1788879272.442405:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304884265 00000100:00000001:3.0:1788879272.442407:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879272.442409:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.442411:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879272.442416:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879272/real 1788879272] req@ffff94e50897a300 x1875765290147072/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879272.442424:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879272.442426:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879272.442428:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879272.442430:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879272.442431:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442433:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879272.442435:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879272.442439:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290147072/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879272.442447:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442450:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879272.442454:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442456:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879272.442457:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442459:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879272.442471:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879272.442474:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879272.442478:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879272.442481:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879272.442484:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290147072:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879272.442488:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442491:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879272.442492:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879272.442494:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879272.442497:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879272.442499:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879272.442500:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879272.442503:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879272.442505:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879272.442510:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879272.442511:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442513:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879272.442515:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879272.442517:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879272.442519:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879272.442520:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879272.442522:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879272.442524:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879272.442525:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879272.442527:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879272.442529:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442530:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879272.442532:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879272.442533:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879272.442535:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879272.442536:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879272.442538:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879272.442539:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879272.442541:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879272.442543:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879272.442545:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442547:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879272.442552:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879272.442554:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879272.442556:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879272.442559:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879272.505047:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879272.505058:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879272.505062:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879272.505081:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879272.505087:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879272.505093:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879272.505097:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879272.505102:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879272.505106:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879273.208987:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879273.209022:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879273.209028:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879273.209032:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879273.209034:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879273.209038:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879273.209041:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879273.209043:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879273.209046:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879273.209048:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879273.209069:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879273.209071:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879273.209073:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879273.528973:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879273.528980:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879273.528982:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879273.528986:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879273.528989:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879273.528991:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879273.528993:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879273.528996:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879273.528998:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879274.232992:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879274.233011:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879274.233016:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879274.233018:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879274.233046:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879274.233048:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879274.233050:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879274.553025:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879274.553034:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879274.553037:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879274.553044:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879274.553048:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879274.553053:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879274.553057:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879274.553061:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879274.553064:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879275.257019:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879275.257027:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879275.257035:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879275.257047:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879275.257084:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879275.257089:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879275.257095:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879275.577066:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879275.577074:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879275.577076:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879275.577083:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879275.577087:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879275.577091:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879275.577093:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879275.577096:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879275.577099:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879276.281040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879276.281050:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879276.281053:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879276.281058:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879276.281064:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879276.281067:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879276.281070:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879276.281073:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879276.281077:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879276.281079:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879276.281104:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879276.281107:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879276.281110:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879276.601035:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879276.601045:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879276.601048:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879276.601055:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879276.601060:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879276.601065:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879276.601068:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879276.601072:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879276.601075:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879277.305088:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879277.305096:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879277.305105:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879277.305108:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879277.305138:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879277.305141:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879277.305144:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879277.561130:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879277.561143:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879277.561147:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879277.561151:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879277.561168:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879277.561177:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10216 00000100:00080000:1.0:1788879277.561190:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879277.561193:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879277.561196:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879277.561206:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879277.561209:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879277.561214:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879277.561217:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879277.561220:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879277.561221:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879277.561227:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879277.561230:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879277.561234:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879277.561238:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879277.561241:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879277.561244:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879277.561247:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879277.561260:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879277.561262:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879277.561266:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879277.561270:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879277.561275:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879277.561290:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879277.561303:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100a680 x1875765290147200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879277.561317:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879277.561323:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879277.561345:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879277.561350:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879277.561354:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879277.561356:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879277.561358:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879277.561362:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879277.561365:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10216 00000100:00080000:1.0:1788879277.561370:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879277.561373:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879277.561375:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879277.561380:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879277.561381:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879277.561384:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879277.561388:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879277.561392:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879277.561393:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879277.561398:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879277.561400:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879277.561401:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879277.561408:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879277.561409:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879277.561415:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1788879277.561416:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879277.561419:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788879277.561426:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.561427:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879277.561438:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788879277.561440:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:1.0:1788879277.561444:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788879277.561446:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:1.0:1788879277.561456:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879277.561458:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879277.561469:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879277.561478:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1788879277.561480:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290147200:192.168.203.129@tcp:250:kworker.0 02000000:00000001:1.0:1788879277.561482:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879277.561489:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879277.561493:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879277.561499:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879277.561500:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879277.561511:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1788879277.561512:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879277.561521:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879277.561527:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b800 x1875765290147328/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879277.561531:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879277.561545:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879277.561546:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879277.561554:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879277.561560:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879277.561563:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879277.561567:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879277.561569:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879277.561571:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879277.561574:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879277.561576:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1788879277.561579:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10216 00000100:00000001:3.0:1788879277.561582:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286517888 : -117763423033728 : ffff94e51100a680) 00000100:00080000:1.0:1788879277.561589:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879277.561591:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879277.561596:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879277.561601:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879277.561606:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879277.561608:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879277.561610:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879277.561613:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879277.561616:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879277.561618:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879277.561623:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879277.561626:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879277.561629:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879277.561633:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879277.561636:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879277.561639:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879277.561645:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788879277.561646:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00000001:1.0:1788879277.561654:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879277.561659:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.561667:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:1.0:1788879277.561668:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879277.561675:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879277.561679:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879277.561679:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1788879277.561686:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879277.561687:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879277.561690:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.561693:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879277.561701:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.561703:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.561712:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304889385 00000100:00100000:1.0:1788879277.561714:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b480 x1875765290147456/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879277.561730:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879277.561730:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788879277.561739:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.561746:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304889385 00000100:00000001:1.0:1788879277.561747:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879277.561756:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879277.561759:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1788879277.561764:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879277.561768:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879277.561770:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879277.561776:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879277.561778:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879277.561782:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879277.561785:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1788879277.561791:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10216 00000100:00080000:1.0:1788879277.561798:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788879277.561800:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879277/real 1788879277] req@ffff94e51100a680 x1875765290147200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879277.561804:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879277.561810:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879277.561824:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000001:3.0:1788879277.561830:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:1.0:1788879277.561831:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000001:3.0:1788879277.561835:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:1.0:1788879277.561839:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879277.561842:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879277.561846:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000020:00000001:1.0:1788879277.561852:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879277.561853:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.561860:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879277.561866:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879277.561868:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1788879277.561875:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100a680 x1875765290147200/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879277.561877:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879277.561897:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788879277.561899:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879277.561937:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879277.561943:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.561946:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879277.561948:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.561951:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879277.561955:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879277.561959:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879277.561968:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879277.561972:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879277.562007:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290147200:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879277.562031:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562036:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788879277.562040:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879277.562045:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1788879277.562052:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879277.562053:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879277.562061:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879277.562063:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879277.562069:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788879277.562073:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000008:00100000:1.0:1788879277.562074:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000020:00000001:3.0:1788879277.562082:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000008:00000001:1.0:1788879277.562083:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879277.562091:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:1.0:1788879277.562092:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879277.562103:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879277.562104:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879277.562110:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879277.562113:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879277.562115:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879277.562121:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879277.562122:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879277.562130:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788879277.562132:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562135:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879277.562144:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562146:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879277.562154:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562158:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879277.562162:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1788879277.562163:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100aa00 x1875765290147584/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879277.562177:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879277.562182:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879277.562187:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879277.562198:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788879277.562200:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788879277.562205:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879277.562206:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879277.562214:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879277.562217:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879277.562232:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290147328:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879277.562240:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879277.562245:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879277.562249:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879277.562253:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879277.562261:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562275:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879277.562279:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:3.0:1788879277.562287:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879277.562309:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879277.562315:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879277.562319:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879277.562321:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562324:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879277.562328:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562333:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304888844 00000100:00000001:3.0:1788879277.562338:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304889385 00000100:00000001:3.0:1788879277.562341:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304888844 00000100:00000001:3.0:1788879277.562344:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304889385 00000100:00000001:3.0:1788879277.562347:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879277.562349:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562352:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879277.562359:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879277/real 1788879277] req@ffff94e51100b800 x1875765290147328/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879277.562370:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879277.562372:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879277.562375:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879277.562377:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879277.562379:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562381:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879277.562383:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879277.562388:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290147328/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879277.562396:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562399:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879277.562402:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562404:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879277.562405:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562408:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879277.562422:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879277.562425:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879277.562430:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879277.562433:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879277.562437:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290147328:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879277.562441:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879277.562443:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879277.562444:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879277.562447:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879277.562449:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879277.562452:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879277.562456:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290147456:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879277.562463:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879277.562466:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879277.562469:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879277.562473:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879277.562477:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562487:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879277.562490:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:3.0:1788879277.562496:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879277.562518:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879277.562526:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879277.562530:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879277.562534:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562537:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879277.562542:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562559:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304888842 00000100:00000001:3.0:1788879277.562565:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304889385 00000100:00000001:3.0:1788879277.562569:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304888842 00000100:00000001:3.0:1788879277.562574:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304889385 00000100:00000001:3.0:1788879277.562576:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879277.562579:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562582:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879277.562588:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879277/real 1788879277] req@ffff94e51100b480 x1875765290147456/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879277.562598:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879277.562600:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879277.562604:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879277.562606:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879277.562608:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562610:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879277.562612:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879277.562617:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290147456/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879277.562634:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562637:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879277.562640:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562642:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879277.562643:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562645:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879277.562657:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879277.562660:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879277.562664:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879277.562666:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879277.562670:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290147456:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879277.562675:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562678:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879277.562679:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879277.562682:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879277.562685:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879277.562687:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879277.562688:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879277.562692:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879277.562693:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879277.562696:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879277.562698:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562700:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879277.562701:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879277.562703:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879277.562705:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879277.562706:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879277.562708:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879277.562710:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879277.562711:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879277.562713:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879277.562714:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562716:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562721:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879277.562723:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562726:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879277.562728:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879277.562729:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879277.562731:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879277.562733:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879277.562735:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879277.562737:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879277.562740:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879277.562743:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290147584:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879277.562747:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879277.562749:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879277.562751:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879277.562754:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879277.562757:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562763:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879277.562765:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:3.0:1788879277.562770:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879277.562785:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879277.562790:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879277.562793:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879277.562795:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562797:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879277.562801:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562805:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304888842 00000100:00000001:3.0:1788879277.562808:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304889385 00000100:00000001:3.0:1788879277.562811:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304888842 00000100:00000001:3.0:1788879277.562814:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304889385 00000100:00000001:3.0:1788879277.562816:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879277.562819:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562821:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879277.562826:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879277/real 1788879277] req@ffff94e51100aa00 x1875765290147584/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879277.562843:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879277.562845:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879277.562847:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879277.562849:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879277.562851:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562853:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879277.562854:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879277.562859:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290147584/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879277.562867:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562869:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879277.562873:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562875:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879277.562877:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562879:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879277.562891:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879277.562894:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879277.562897:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879277.562899:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879277.562933:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290147584:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879277.562938:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562941:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879277.562943:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879277.562945:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879277.562948:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879277.562949:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879277.562951:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879277.562953:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879277.562955:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879277.562957:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879277.562959:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562962:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879277.562965:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879277.562967:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879277.562970:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879277.562972:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879277.625045:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879277.625052:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879277.625055:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879277.625061:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879277.625064:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879277.625069:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879277.625072:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879277.625076:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879277.625079:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879278.329084:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879278.329095:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879278.329102:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879278.329103:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879278.329137:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879278.329158:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879278.329162:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879278.649063:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879278.649071:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879278.649073:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879278.649079:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879278.649083:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879278.649087:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879278.649090:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879278.649093:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879278.649096:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879279.353056:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879279.353065:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879279.353076:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879279.353079:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879279.353112:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879279.353118:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879279.353123:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879279.673088:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879279.673097:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879279.673108:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879279.673116:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879279.673121:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879279.673126:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879279.673129:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879279.673133:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879279.673136:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879280.377006:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879280.377011:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879280.377015:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879280.377016:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879280.377036:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879280.377039:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879280.377042:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879280.697096:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879280.697105:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879280.697108:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879280.697114:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879280.697119:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879280.697124:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879280.697127:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879280.697131:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879280.697134:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879281.401059:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879281.401065:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879281.401070:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879281.401071:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879281.401090:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879281.401093:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879281.401095:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879281.721028:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879281.721036:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879281.721040:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879281.721063:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879281.721070:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879281.721075:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879281.721080:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879281.721084:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879281.721089:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879282.425051:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879282.425059:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879282.425066:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879282.425067:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879282.425094:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879282.425098:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879282.425101:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879282.681110:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879282.681121:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879282.681124:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879282.681139:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879282.681144:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879282.681152:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10221 00000100:00080000:2.0:1788879282.681163:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879282.681167:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879282.681169:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879282.681177:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879282.681180:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879282.681183:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879282.681186:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879282.681188:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879282.681189:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879282.681193:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879282.681196:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879282.681198:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879282.681202:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879282.681205:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681207:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879282.681210:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879282.681222:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879282.681223:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681227:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879282.681229:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879282.681234:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681246:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879282.681255:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4380 x1875765290147712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879282.681266:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879282.681271:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681289:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879282.681292:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879282.681296:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879282.681297:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879282.681299:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879282.681302:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879282.681304:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10221 00000100:00080000:2.0:1788879282.681308:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879282.681310:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879282.681312:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879282.681315:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879282.681316:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879282.681318:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879282.681320:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879282.681322:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879282.681324:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879282.681327:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879282.681330:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879282.681332:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879282.681338:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879282.681340:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681342:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879282.681346:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879282.681350:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681352:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879282.681355:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879282.681356:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681359:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879282.681359:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879282.681361:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879282.681365:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.681370:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879282.681371:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.681378:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1788879282.681380:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5f80 x1875765290147840/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879282.681385:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879282.681392:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681394:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879282.681401:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879282.681404:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879282.681408:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879282.681412:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788879282.681415:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1788879282.681422:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879282.681425:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879282.681432:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1788879282.681433:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290147712:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1788879282.681439:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879282.681442:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788879282.681449:0:136860:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:3.0:1788879282.681451:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879282.681456:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879282.681459:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10221 02000000:00000001:3.0:1788879282.681460:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879282.681470:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879282.681471:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879282.681477:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879282.681478:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879282.681483:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879282.681485:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879282.681487:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879282.681490:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879282.681491:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879282.681492:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879282.681496:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074549120 : -117763635002496 : ffff94e5045e4380) 00000100:00000001:2.0:1788879282.681497:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879282.681507:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879282.681508:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879282.681510:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879282.681515:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879282.681518:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879282.681521:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681523:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879282.681526:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879282.681528:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681530:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879282.681537:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879282.681538:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681540:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879282.681542:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879282.681544:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681550:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879282.681556:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290147968/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879282.681565:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879282.681568:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681573:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879282.681577:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879282.681581:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879282.681584:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879282.681587:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879282.681589:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681593:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879282.681597:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304894505 00000100:00080000:2.0:1788879282.681601:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10221 00000100:00080000:2.0:1788879282.681611:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879282.681612:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788879282.681615:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879282.681620:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879282.681623:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304894505 00000020:00000001:2.0:1788879282.681626:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879282.681631:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1788879282.681632:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879282.681639:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879282.681640:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1788879282.681644:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879282.681647:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681649:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879282.681653:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681654:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879282.681661:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879282.681663:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879282.681667:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879282.681669:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879282.681670:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1788879282.681674:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879282.681675:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:2.0:1788879282.681682:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.681684:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879282.681687:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879282.681689:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1788879282.681691:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879282.681694:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00000001:2.0:1788879282.681696:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.681698:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879282.681702:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879282.681705:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879282.681708:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879282.681710:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681712:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879282.681716:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290147840:192.168.203.129@tcp:38:kworker.0 02000000:00000001:2.0:1788879282.681721:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879282.681722:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879282.681725:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879282.681728:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788879282.681730:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879282.681732:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879282.681734:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879282.681737:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879282.681739:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879282.681744:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681767:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.681777:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00100000:2.0:1788879282.681778:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290148096/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879282.681783:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074556288 : -117763634995328 : ffff94e5045e5f80) 00000100:00000001:3.0:1788879282.681794:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879282.681798:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879282.681803:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879282.681809:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.681828:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.681837:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304893963 00000100:00000001:3.0:1788879282.681841:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304894505 00000100:00000001:3.0:1788879282.681845:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304893963 00000100:00000001:3.0:1788879282.681848:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304894505 00000100:00000001:3.0:1788879282.681851:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879282.681854:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.681856:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879282.681859:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879282.681861:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879282.681864:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879282.681867:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879282.681869:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879282.681875:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290147968:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879282.681878:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879282.681880:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879282.681881:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879282.681883:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879282.681885:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.681891:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879282.681893:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000100:00000001:3.0:1788879282.681896:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879282.681908:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.681913:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304893961 00000100:00000001:3.0:1788879282.681917:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304894505 00000100:00000001:3.0:1788879282.682003:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304893962 00000100:00000001:3.0:1788879282.682007:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304894505 00000100:00000001:3.0:1788879282.682011:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879282.682014:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.682018:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.682021:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879282.682026:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879282.682029:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788879282.682031:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879282.682033:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879282.682034:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879282.682035:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879282.682037:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879282.682039:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879282.682042:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879282.682044:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879282.682048:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290148096:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879282.682051:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879282.682053:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879282.682055:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879282.682057:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879282.682060:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.682067:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879282.682070:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 00000100:00000001:3.0:1788879282.682075:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879282.682092:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.682098:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304893962 00000100:00000001:3.0:1788879282.682102:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304894505 00000100:00000001:3.0:1788879282.682106:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304893962 00000100:00000001:3.0:1788879282.682109:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304894505 00000100:00000001:3.0:1788879282.682113:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879282.682116:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.682120:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.682122:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879282.682126:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879282.682128:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788879282.682131:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879282.682133:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879282.682135:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879282.682138:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879282.745032:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879282.745041:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879282.745044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879282.745050:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879282.745055:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879282.745059:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879282.745062:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879282.745065:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879282.745068:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879283.449082:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879283.449092:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879283.449100:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879283.449104:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879283.449109:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879283.449114:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879283.449118:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879283.449121:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879283.449124:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879283.449127:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879283.449160:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879283.449164:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879283.449168:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879283.769149:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879283.769165:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879283.769170:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879283.769182:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879283.769191:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879283.769199:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879283.769205:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879283.769211:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879283.769216:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879284.473077:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879284.473088:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879284.473098:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879284.473101:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879284.473136:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879284.473141:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879284.473147:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879284.793050:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879284.793059:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879284.793062:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879284.793069:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879284.793074:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879284.793080:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879284.793083:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879284.793086:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879284.793089:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879285.497081:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879285.497092:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879285.497101:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879285.497104:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879285.497140:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879285.497145:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879285.497151:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1788879285.753454:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:2.0:1788879285.753514:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1788879285.753536:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788879285.753553:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879285.753578:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879285.753581:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879285.753584:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:2.0:1788879285.753596:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788879285.753601:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879285.753604:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879285.753606:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879285.753608:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:2.0:1788879285.753613:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788879285.753619:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879285.753623:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879285.753624:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879285.753626:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:2.0:1788879285.753631:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:2.0:1788879285.753638:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879285.753642:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879285.753644:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879285.753648:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879285.753669:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879285.753679:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879285.753684:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879285.753714:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879282/real 1788879285] req@ffff94e5045e6300 x1875765290148096/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879285.753742:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879285.753745:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879285.753756:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879285.753761:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879285.753764:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.753769:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879285.753773:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879285.753782:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290148096/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879285.753799:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.753806:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879285.753811:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.753814:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879285.753817:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.753822:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879285.753846:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879285.753855:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879285.753864:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879285.753869:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879285.753876:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290148096:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879285.753885:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879285.753894:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879282/real 1788879285] req@ffff94e5045e5f80 x1875765290147840/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879285.754010:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879285.754014:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879285.754023:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879285.754029:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879285.754033:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.754037:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879285.754040:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879285.754051:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5f80 x1875765290147840/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879285.754068:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.754073:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879285.754078:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.754081:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879285.754083:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.754086:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879285.754107:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879285.754113:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879285.754120:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879285.754125:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879285.754131:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290147840:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879285.754140:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879285.754149:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879282/real 1788879285] req@ffff94e5045e7800 x1875765290147968/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879285.754167:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879285.754170:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879285.754175:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879285.754178:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879285.754181:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.754185:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879285.754187:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879285.754196:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290147968/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879285.754210:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.754213:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879285.754218:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.754221:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879285.754224:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.754227:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879285.754248:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879285.754254:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879285.754261:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879285.754266:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879285.754273:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290147968:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879285.754281:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879285.754289:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879282/real 1788879285] req@ffff94e5045e4380 x1875765290147712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879285.754307:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879285.754309:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879285.754315:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879285.754319:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879285.754321:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.754326:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879285.754329:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879285.754337:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4380 x1875765290147712/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879285.754352:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.754356:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879285.754361:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.754364:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879285.754366:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.754369:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879285.754372:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879285.754377:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879285.754384:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879285.754389:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879285.754394:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290147712:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879285.754402:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.754406:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879285.754409:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879285.754414:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879285.754423:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879285.755216:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879285.755222:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879285.755230:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879285.755233:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879285.755239:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879285.755242:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.755247:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879285.755250:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879285.755253:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879285.755257:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879285.755259:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879285.755261:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879285.755264:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879285.755267:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879285.755270:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879285.755274:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.755277:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879285.755280:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879285.755283:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879285.755286:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879285.755288:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879285.755291:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879285.755294:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879285.755296:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879285.755299:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879285.755302:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.755305:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879285.755307:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879285.755310:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879285.755313:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879285.755315:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879285.755318:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879285.755321:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879285.755323:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879285.755326:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879285.755329:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.755334:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879285.755345:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879285.755349:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879285.755353:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879285.755356:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879285.817063:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879285.817071:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879285.817074:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879285.817080:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879285.817085:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879285.817089:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879285.817093:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879285.817096:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879285.817099:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879286.520989:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879286.520995:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879286.520997:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879286.521000:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879286.521003:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879286.521005:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879286.521006:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879286.521008:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879286.521010:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879286.521011:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879286.521028:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879286.521030:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879286.521032:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879286.840976:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879286.840982:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879286.840984:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879286.840988:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879286.840991:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879286.840994:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879286.840996:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879286.840999:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879286.841001:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879287.545086:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879287.545099:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879287.545111:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879287.545115:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879287.545155:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879287.545160:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879287.545164:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879287.801181:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879287.801194:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879287.801198:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879287.801203:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879287.801210:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879287.801222:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10226 00000100:00080000:0.0:1788879287.801237:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879287.801241:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879287.801244:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879287.801257:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879287.801261:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879287.801266:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879287.801269:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879287.801272:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879287.801274:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879287.801281:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879287.801284:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879287.801288:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879287.801293:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879287.801298:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879287.801302:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879287.801306:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879287.801318:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879287.801321:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879287.801325:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879287.801330:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879287.801338:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879287.801355:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879287.801369:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978700 x1875765290148224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879287.801384:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879287.801391:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879287.801415:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879287.801422:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879287.801429:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879287.801432:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879287.801436:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879287.801442:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879287.801448:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10226 00000100:00080000:0.0:1788879287.801456:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879287.801461:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879287.801464:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879287.801471:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879287.801474:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879287.801478:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879287.801482:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879287.801487:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879287.801490:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879287.801499:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879287.801504:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879287.801508:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879287.801522:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879287.801525:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788879287.801530:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.801534:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879287.801538:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879287.801541:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879287.801547:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:0.0:1788879287.801551:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879287.801552:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788879287.801566:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879287.801567:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788879287.801573:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879287.801578:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879287.801586:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879287.801588:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879287.801594:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879287.801603:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1788879287.801607:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290148224:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788879287.801609:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879287.801617:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.801625:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788879287.801634:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879287.801638:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879287.801647:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879287.801654:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897aa00 x1875765290148352/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879287.801656:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879287.801670:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879287.801679:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879287.801684:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879287.801696:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879287.801702:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879287.801710:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879287.801713:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879287.801717:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879287.801723:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879287.801724:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879287.801733:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00080000:0.0:1788879287.801737:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10226 00000100:00000001:3.0:1788879287.801748:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788879287.801752:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879287.801761:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879287.801765:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879287.801772:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879287.801775:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879287.801781:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879287.801786:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879287.801791:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879287.801794:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879287.801804:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879287.801811:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879287.801815:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879287.801822:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879287.801826:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879287.801830:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879287.801835:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788879287.801842:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00000001:0.0:1788879287.801845:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879287.801854:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879287.801865:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879287.801868:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.801870:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879287.801874:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879287.801881:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879287.801883:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1788879287.801886:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.801892:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.801900:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879287.801994:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.802008:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879287.802013:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290148480/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879287.802022:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304899626 00000100:00000001:0.0:1788879287.802044:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879287.802049:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.802051:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788879287.802056:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879287.802066:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879287.802069:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304899626 00000100:00080000:0.0:1788879287.802077:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879287.802084:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788879287.802086:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:0.0:1788879287.802089:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879287.802101:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879287.802107:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10226 00000100:00000001:3.0:1788879287.802109:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879287.802121:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879287.802122:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879287.802131:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879287.802133:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879287.802140:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879287.802147:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:3.0:1788879287.802151:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879287/real 1788879287] req@ffff94e508978700 x1875765290148224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788879287.802153:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879287.802163:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879287.802171:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879287.802176:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879287.802179:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879287.802188:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879287.802206:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879287.802211:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879287.802218:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879287.802222:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879287.802226:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879287.802231:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879287.802236:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879287.802239:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879287.802268:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879287.802271:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879287.802276:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879287.802278:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879287.802282:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879287.802294:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879287.802304:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290148608/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879287.802320:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879287.802324:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879287.802329:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879287.802454:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879287.802459:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879287.802468:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879287.802476:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879287.802481:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.802486:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879287.802491:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879287.802506:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290148224/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879287.802522:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.802529:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879287.802536:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.802540:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879287.802542:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.802547:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879287.802553:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879287.802559:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879287.802568:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879287.802573:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879287.802580:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290148224:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879287.802591:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.802596:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879287.802599:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879287.802604:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879287.802611:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879287.802615:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879287.802619:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879287.802626:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879287.802630:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879287.802635:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879287.802638:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.802644:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.802659:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879287.802663:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.802668:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879287.802671:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879287.802674:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879287.802678:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879287.802681:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879287.802687:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879287.802691:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879287.802695:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879287.802703:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290148352:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879287.802711:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879287.802715:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879287.802718:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879287.802722:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879287.802729:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.802742:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879287.802746:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:3.0:1788879287.802756:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879287.802788:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879287.802798:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879287.802803:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879287.802806:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.802809:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879287.802815:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.802823:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304899084 00000100:00000001:3.0:1788879287.802828:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304899626 00000100:00000001:3.0:1788879287.802833:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304899084 00000100:00000001:3.0:1788879287.802837:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304899626 00000100:00000001:3.0:1788879287.802841:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879287.802845:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.802849:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879287.802858:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879287/real 1788879287] req@ffff94e50897aa00 x1875765290148352/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879287.802876:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879287.802878:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879287.802884:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879287.802887:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879287.802890:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.802894:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879287.802897:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879287.802989:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290148352/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879287.803008:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.803014:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879287.803019:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.803022:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879287.803025:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.803028:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879287.803055:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879287.803063:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879287.803073:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879287.803080:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879287.803088:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290148352:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879287.803099:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879287.803120:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879287.803125:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879287.803140:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879287.803145:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879287.803149:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879287.803157:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290148480:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879287.803164:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879287.803168:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879287.803171:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879287.803175:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879287.803181:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.803195:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879287.803198:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:3.0:1788879287.803206:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879287.803234:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879287.803243:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879287.803248:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879287.803251:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.803254:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879287.803260:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.803267:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304899083 00000100:00000001:3.0:1788879287.803272:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304899626 00000100:00000001:3.0:1788879287.803277:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304899083 00000100:00000001:3.0:1788879287.803282:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304899626 00000100:00000001:3.0:1788879287.803287:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879287.803291:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.803296:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879287.803304:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879287/real 1788879287] req@ffff94e508978a80 x1875765290148480/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879287.803316:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879287.803318:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879287.803322:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879287.803324:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879287.803326:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.803329:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879287.803331:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879287.803336:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290148480/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879287.803345:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.803348:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879287.803351:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.803354:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879287.803355:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.803357:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879287.803376:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879287.803379:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879287.803383:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879287.803385:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879287.803389:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290148480:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879287.803394:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879287.803396:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879287.803398:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879287.803401:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879287.803404:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879287.803406:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879287.803410:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290148608:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879287.803418:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879287.803420:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879287.803423:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879287.803425:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879287.803429:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.803436:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879287.803438:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:3.0:1788879287.803443:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879287.803459:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879287.803465:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879287.803468:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879287.803470:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.803473:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879287.803478:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.803486:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304899083 00000100:00000001:3.0:1788879287.803491:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304899626 00000100:00000001:3.0:1788879287.803496:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304899083 00000100:00000001:3.0:1788879287.803500:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304899626 00000100:00000001:3.0:1788879287.803504:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879287.803507:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.803511:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879287.803519:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879287/real 1788879287] req@ffff94e50897a300 x1875765290148608/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879287.803537:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879287.803540:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879287.803546:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879287.803549:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879287.803553:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.803556:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879287.803559:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879287.803568:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290148608/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879287.803593:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.803597:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879287.803600:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.803602:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879287.803604:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.803607:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879287.803625:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879287.803629:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879287.803633:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879287.803636:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879287.803640:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290148608:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879287.803645:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.803648:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879287.803650:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879287.803653:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879287.803656:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879287.803658:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879287.803660:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879287.803664:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879287.803666:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879287.803671:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879287.803672:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.803675:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879287.803676:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879287.803678:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879287.803680:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879287.803682:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879287.803683:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879287.803686:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879287.803688:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879287.803690:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879287.803692:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.803694:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879287.803695:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879287.803698:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879287.803700:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879287.803702:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879287.803703:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879287.803705:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879287.803707:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879287.803709:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879287.803710:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.803712:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879287.803718:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879287.803720:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879287.803722:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879287.803724:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879287.865080:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879287.865090:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879287.865094:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879287.865103:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879287.865108:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879287.865112:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879287.865116:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879287.865119:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879287.865123:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879288.569053:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879288.569061:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879288.569084:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879288.569086:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879288.569115:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879288.569118:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879288.569122:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879288.888995:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879288.889003:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879288.889007:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879288.889015:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879288.889039:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879288.889045:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879288.889050:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879288.889055:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879288.889061:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879289.593009:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879289.593015:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879289.593020:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879289.593021:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879289.593040:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879289.593042:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879289.593045:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879289.913098:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879289.913106:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879289.913109:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879289.913115:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879289.913120:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879289.913124:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879289.913127:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879289.913147:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879289.913161:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879290.617034:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879290.617041:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879290.617046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879290.617048:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879290.617071:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879290.617074:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879290.617078:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879290.937092:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879290.937105:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879290.937110:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879290.937121:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879290.937129:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879290.937136:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879290.937143:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879290.937149:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879290.937154:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879291.641062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879291.641071:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879291.641077:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879291.641078:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879291.641105:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879291.641108:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879291.641123:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879291.961057:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879291.961062:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879291.961064:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879291.961068:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879291.961070:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879291.961073:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879291.961075:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879291.961077:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879291.961079:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879292.665036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879292.665042:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879292.665047:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879292.665049:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879292.665071:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879292.665074:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879292.665076:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879292.921064:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879292.921075:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879292.921078:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879292.921083:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879292.921106:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879292.921119:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10231 00000100:00080000:0.0:1788879292.921138:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879292.921142:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879292.921145:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879292.921156:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879292.921160:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879292.921164:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879292.921167:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879292.921169:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879292.921171:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879292.921176:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879292.921179:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879292.921183:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879292.921187:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879292.921191:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921194:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879292.921197:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879292.921234:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879292.921237:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921242:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879292.921246:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879292.921254:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921273:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879292.921287:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290148736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879292.921303:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879292.921309:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921342:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879292.921348:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879292.921352:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879292.921354:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879292.921357:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879292.921362:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879292.921365:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10231 00000100:00080000:0.0:1788879292.921370:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879292.921373:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879292.921375:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879292.921379:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879292.921381:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879292.921384:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879292.921386:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879292.921390:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879292.921392:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879292.921396:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879292.921399:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879292.921401:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879292.921410:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879292.921412:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921414:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879292.921420:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879292.921424:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921427:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879292.921432:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879292.921434:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921437:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879292.921439:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879292.921441:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921448:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879292.921456:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290148864/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879292.921483:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879292.921486:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921487:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879292.921489:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879292.921493:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879292.921493:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879292.921497:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879292.921499:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879292.921500:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879292.921501:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879292.921504:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1788879292.921506:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10231 00000100:00000001:3.0:1788879292.921509:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788879292.921510:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879292.921512:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921513:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879292.921516:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879292.921519:0:80852:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:3.0:1788879292.921520:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1788879292.921521:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:3.0:1788879292.921524:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788879292.921525:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879292.921531:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 02000000:00000001:3.0:1788879292.921532:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921534:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1788879292.921540:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290148736:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879292.921544:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879292.921547:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1788879292.921551:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879292.921553:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879292.921554:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879292.921554:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879292.921559:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879292.921560:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879292.921565:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879292.921566:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879292.921568:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879292.921571:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788879292.921572:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788879292.921573:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921578:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879292.921582:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879292.921583:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921584:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879292.921585:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879292.921587:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921592:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879292.921596:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290148992/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879292.921597:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879292.921600:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:3.0:1788879292.921606:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879292.921614:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879292.921616:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921618:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879292.921620:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879292.921622:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879292.921623:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879292.921625:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879292.921627:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879292.921629:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10231 00000100:00080000:0.0:1788879292.921631:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879292.921632:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879292.921633:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879292.921635:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879292.921636:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879292.921637:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879292.921639:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879292.921641:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879292.921643:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879292.921645:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879292.921646:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879292.921647:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879292.921649:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879292.921651:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921652:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879292.921653:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879292.921654:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921656:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788879292.921660:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:0.0:1788879292.921661:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879292.921663:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921665:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879292.921667:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879292.921668:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921672:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879292.921675:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290149120/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879292.921676:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879292.921682:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879292.921683:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879292.921685:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.921686:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.921691:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879292.921692:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.921699:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.921704:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304904745 00000100:00000001:3.0:1788879292.921716:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879292.921724:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304904745 00000100:00000001:3.0:1788879292.921728:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879292.921732:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879292.921735:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.921738:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879292.921744:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879292/real 1788879292] req@ffff94e508979c00 x1875765290148736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879292.921755:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879292.921756:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879292.921759:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879292.921761:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879292.921763:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.921765:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879292.921767:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879292.921770:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290148736/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879292.921776:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.921779:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879292.921782:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.921783:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879292.921784:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.921786:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879292.921788:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879292.921790:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879292.921793:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879292.921795:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879292.921797:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290148736:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879292.921800:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879292.921801:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879292.921802:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879292.921803:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879292.921805:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879292.921807:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879292.921809:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290148864:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879292.921812:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879292.921813:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879292.921814:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879292.921815:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879292.921817:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.921820:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879292.921821:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:3.0:1788879292.921825:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879292.921832:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879292.921835:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879292.921836:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879292.921837:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.921839:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879292.921840:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.921844:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304904203 00000100:00000001:3.0:1788879292.921846:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304904745 00000100:00000001:3.0:1788879292.921848:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304904203 00000100:00000001:3.0:1788879292.921850:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304904745 00000100:00000001:3.0:1788879292.921851:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879292.921853:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.921854:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879292.921856:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879292/real 1788879292] req@ffff94e50897ad80 x1875765290148864/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879292.921861:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879292.921862:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879292.921863:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879292.921864:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879292.921865:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.921866:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879292.921867:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879292.921887:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290148864/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879292.921912:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.921914:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879292.921943:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.921946:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879292.921947:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.921949:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879292.921970:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879292.921973:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879292.921977:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879292.921978:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879292.921981:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290148864:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879292.921985:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.921987:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879292.921988:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879292.921992:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879292.921995:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879292.921997:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879292.921999:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879292.922003:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879292.922005:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879292.922008:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879292.922010:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.922027:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879292.922027:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879292.922028:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879292.922030:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879292.922031:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879292.922031:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879292.922032:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879292.922033:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879292.922036:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879292.922037:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.922039:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.922047:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879292.922049:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.922050:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879292.922052:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879292.922053:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879292.922054:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879292.922055:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879292.922057:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879292.922058:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879292.922060:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879292.922062:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290148992:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879292.922064:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879292.922066:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879292.922067:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879292.922068:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879292.922071:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.922076:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879292.922077:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:3.0:1788879292.922081:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879292.922091:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879292.922111:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879292.922114:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879292.922115:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.922117:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879292.922120:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.922125:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304904202 00000100:00000001:3.0:1788879292.922127:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304904745 00000100:00000001:3.0:1788879292.922130:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304904202 00000100:00000001:3.0:1788879292.922132:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304904745 00000100:00000001:3.0:1788879292.922135:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879292.922137:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.922139:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879292.922143:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879292/real 1788879292] req@ffff94e508978e00 x1875765290148992/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879292.922159:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879292.922160:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879292.922163:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879292.922165:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879292.922166:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.922168:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879292.922170:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879292.922174:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290148992/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879292.922199:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.922202:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879292.922205:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.922207:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879292.922208:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.922210:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879292.922230:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879292.922232:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879292.922236:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879292.922238:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879292.922242:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290148992:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879292.922246:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879292.922248:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879292.922250:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879292.922253:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879292.922255:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879292.922258:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879292.922261:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290149120:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879292.922265:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879292.922267:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879292.922270:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879292.922272:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879292.922275:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.922281:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879292.922283:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:3.0:1788879292.922288:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879292.922301:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879292.922306:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879292.922308:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879292.922310:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.922313:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879292.922316:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.922320:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304904202 00000100:00000001:3.0:1788879292.922323:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304904745 00000100:00000001:3.0:1788879292.922326:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304904202 00000100:00000001:3.0:1788879292.922329:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304904745 00000100:00000001:3.0:1788879292.922332:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879292.922334:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.922336:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879292.922340:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879292/real 1788879292] req@ffff94e50897bb80 x1875765290149120/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879292.922349:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879292.922350:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879292.922353:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879292.922354:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879292.922356:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.922358:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879292.922359:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879292.922363:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290149120/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879292.922371:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.922374:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879292.922378:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.922380:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879292.922382:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.922384:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879292.922402:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879292.922405:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879292.922410:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879292.922413:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879292.922417:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290149120:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879292.922421:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.922423:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879292.922424:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879292.922426:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879292.922428:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879292.922430:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879292.922431:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879292.922434:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879292.922435:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879292.922438:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879292.922439:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.922441:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879292.922443:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879292.922445:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879292.922446:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879292.922448:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879292.922449:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879292.922452:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879292.922453:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879292.922455:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879292.922456:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.922459:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879292.922462:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879292.922464:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879292.922466:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879292.922468:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879292.985031:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879292.985040:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879292.985042:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879292.985047:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879292.985050:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879292.985054:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879292.985057:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879292.985059:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879292.985061:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879293.688970:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879293.688977:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879293.688983:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879293.688986:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879293.688988:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879293.688992:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879293.688995:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879293.688998:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879293.689000:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879293.689002:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879293.689024:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879293.689028:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879293.689031:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879294.009080:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879294.009091:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879294.009094:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879294.009101:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879294.009106:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879294.009111:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879294.009115:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879294.009119:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879294.009122:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879294.713111:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879294.713120:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879294.713127:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879294.713129:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879294.713163:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879294.713167:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879294.713171:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879295.033087:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879295.033100:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879295.033105:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879295.033114:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879295.033121:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879295.033128:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879295.033133:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879295.033138:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879295.033143:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879295.737052:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879295.737062:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879295.737071:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879295.737074:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879295.737130:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879295.737135:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879295.737141:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879296.056981:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879296.056986:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879296.056988:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879296.056993:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879296.056996:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879296.056999:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879296.057002:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879296.057004:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879296.057007:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879296.761027:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879296.761035:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879296.761038:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879296.761042:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879296.761047:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879296.761049:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879296.761051:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879296.761053:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879296.761057:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879296.761058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879296.761102:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879296.761105:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879296.761108:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879297.081054:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879297.081061:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879297.081064:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879297.081070:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879297.081075:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879297.081079:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879297.081083:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879297.081086:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879297.081089:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879297.785058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879297.785066:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879297.785072:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879297.785074:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879297.785128:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879297.785132:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879297.785136:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879298.041066:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879298.041081:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879298.041085:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879298.041090:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879298.041097:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879298.041107:0:136860:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10236 00000100:00080000:2.0:1788879298.041122:0:136860:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879298.041127:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879298.041129:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879298.041140:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879298.041144:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879298.041148:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879298.041152:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879298.041154:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879298.041156:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879298.041161:0:136860:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879298.041165:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879298.041169:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879298.041174:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879298.041178:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879298.041181:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879298.041184:0:136860:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879298.041200:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879298.041202:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879298.041207:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879298.041211:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879298.041217:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879298.041234:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879298.041246:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290149248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879298.041259:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879298.041265:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879298.041287:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879298.041291:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879298.041295:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879298.041296:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879298.041298:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879298.041302:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879298.041305:0:136860:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10236 00000100:00080000:2.0:1788879298.041311:0:136860:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879298.041315:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879298.041317:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879298.041321:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879298.041323:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879298.041326:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879298.041328:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879298.041331:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879298.041333:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879298.041338:0:136860:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879298.041341:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879298.041346:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879298.041346:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879298.041370:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879298.041375:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788879298.041377:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.041381:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879298.041384:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879298.041386:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879298.041391:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788879298.041394:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788879298.041401:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:2.0:1788879298.041404:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879298.041407:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879298.041411:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879298.041414:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879298.041422:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1788879298.041430:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290149248:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879298.041431:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879298.041442:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879298.041444:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879298.041450:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879298.041453:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879298.041458:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879298.041459:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879298.041469:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879298.041476:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879298.041481:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879298.041492:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290149376/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879298.041504:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879298.041505:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879298.041508:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:2.0:1788879298.041515:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.041518:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879298.041527:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879298.041534:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879298.041539:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879298.041542:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879298.041545:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879298.041551:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879298.041557:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10236 00000100:00080000:2.0:1788879298.041563:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879298.041567:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000800:00000100:3.0:1788879298.041577:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879298.041578:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879298.041591:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879298.041594:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879298.041598:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879298.041598:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:2.0:1788879298.041605:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879298.041606:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879298.041611:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879298.041614:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879298.041615:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879298.041623:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879298.041627:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.041637:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304909865 00000100:00080000:2.0:1788879298.041642:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879298.041649:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.041652:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1788879298.041653:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879298.041664:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879298.041665:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304909865 02000000:00000001:2.0:1788879298.041681:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.041682:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879298.041691:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879298.041695:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879298.041699:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879298.041701:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879298.041707:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879298.041708:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879298.041712:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879298.041712:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879298.041723:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879298.041724:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879298.041728:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1788879298.041730:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879298.041743:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879298/real 1788879298] req@ffff94e5045e5500 x1875765290149248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879298.041745:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879298.041769:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290149504/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788879298.041772:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879298.041776:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879298.041785:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879298.041790:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879298.041795:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879298.041795:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.041803:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879298.041805:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.041807:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879298.041824:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879298.041826:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290149248/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879298.041851:0:136860:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879298.041868:0:136860:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879298.041871:0:136860:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879298.041884:0:136860:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879298.041890:0:136860:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879298.041896:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879298.041898:0:136860:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10236 00000100:00000001:3.0:1788879298.041906:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879298.041913:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.041955:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879298.041959:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879298.041970:0:136860:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879298.041970:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879298.041980:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879298.041980:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879298.041986:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879298.041993:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:2.0:1788879298.041996:0:136860:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879298.042002:0:136860:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879298.042007:0:136860:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00080000:3.0:1788879298.042008:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879298.042016:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:2.0:1788879298.042017:0:136860:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:3.0:1788879298.042025:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290149248:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879298.042027:0:136860:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879298.042037:0:136860:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879298.042039:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879298.042051:0:136860:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879298.042057:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879298.042062:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788879298.042065:0:136860:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879298.042067:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879298.042074:0:136860:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879298.042074:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879298.042082:0:2361:0:(genops.c:1216:class_import_put()) Process entered 02000000:00000001:2.0:1788879298.042087:0:136860:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788879298.042088:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879298.042097:0:136860:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879298.042098:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879298.042101:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879298.042108:0:136860:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879298.042108:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879298.042115:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1788879298.042118:0:136860:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879298.042120:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1788879298.042130:0:136860:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042134:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879298.042135:0:136860:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042138:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042143:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879298.042149:0:136860:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879298.042150:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879298.042156:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788879298.042158:0:136860:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042160:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879298.042163:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879298.042166:0:136860:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879298.042168:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879298.042174:0:136860:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879298.042175:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879298.042185:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879298.042186:0:136860:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879298.042192:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290149376:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879298.042198:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788879298.042199:0:136860:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879298.042201:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879298.042205:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879298.042209:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879298.042212:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879298.042215:0:136860:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290149632/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879298.042223:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879298.042227:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:2.0:1788879298.042231:0:136860:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879298.042234:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879298.042239:0:136860:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879298.042247:0:136860:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788879298.042256:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879298.042264:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879298.042269:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879298.042270:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042272:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879298.042276:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042282:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304909324 00000100:00000001:3.0:1788879298.042285:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304909865 00000100:00000001:3.0:1788879298.042288:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304909324 00000100:00000001:3.0:1788879298.042291:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304909865 00000100:00000001:3.0:1788879298.042293:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879298.042295:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042297:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879298.042302:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879298/real 1788879298] req@ffff94e5045e5c00 x1875765290149376/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879298.042310:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879298.042312:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879298.042314:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879298.042316:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879298.042317:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042319:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879298.042321:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879298.042325:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290149376/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879298.042333:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042335:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879298.042337:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042339:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879298.042340:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042342:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879298.042355:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879298.042358:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879298.042361:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879298.042364:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879298.042367:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290149376:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879298.042371:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879298.042372:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879298.042374:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879298.042376:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879298.042378:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879298.042380:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879298.042383:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290149504:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879298.042386:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879298.042388:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879298.042390:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879298.042391:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879298.042395:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042399:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879298.042401:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:3.0:1788879298.042404:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879298.042414:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879298.042418:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879298.042421:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879298.042422:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042424:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879298.042427:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042430:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304909322 00000100:00000001:3.0:1788879298.042432:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304909865 00000100:00000001:3.0:1788879298.042435:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304909322 00000100:00000001:3.0:1788879298.042437:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304909865 00000100:00000001:3.0:1788879298.042439:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879298.042441:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042442:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879298.042446:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879298/real 1788879298] req@ffff94e5045e7100 x1875765290149504/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879298.042454:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879298.042455:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879298.042457:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879298.042458:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879298.042459:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042461:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879298.042462:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879298.042466:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290149504/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879298.042472:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042474:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879298.042477:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042479:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879298.042480:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042481:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879298.042492:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879298.042494:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879298.042497:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879298.042500:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879298.042503:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290149504:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879298.042507:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042509:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879298.042510:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879298.042512:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879298.042514:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879298.042515:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879298.042516:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879298.042518:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879298.042519:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879298.042521:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879298.042522:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042524:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879298.042525:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879298.042527:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879298.042528:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879298.042530:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879298.042531:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879298.042533:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879298.042534:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879298.042535:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879298.042536:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042538:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042541:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879298.042542:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042544:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879298.042546:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879298.042547:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879298.042548:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879298.042550:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879298.042552:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879298.042554:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879298.042556:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879298.042559:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290149632:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879298.042563:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879298.042564:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879298.042567:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879298.042568:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879298.042571:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042575:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879298.042576:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:3.0:1788879298.042580:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879298.042589:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879298.042593:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879298.042595:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879298.042596:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042598:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879298.042600:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042603:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304909322 00000100:00000001:3.0:1788879298.042606:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304909865 00000100:00000001:3.0:1788879298.042608:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304909322 00000100:00000001:3.0:1788879298.042610:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304909865 00000100:00000001:3.0:1788879298.042613:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879298.042614:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042616:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879298.042620:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879298/real 1788879298] req@ffff94e5045e4a80 x1875765290149632/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879298.042627:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879298.042628:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879298.042630:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879298.042631:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879298.042633:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042634:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879298.042636:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879298.042640:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290149632/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879298.042646:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042648:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879298.042651:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042653:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879298.042654:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042656:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879298.042666:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879298.042669:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879298.042672:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879298.042674:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879298.042678:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290149632:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879298.042681:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042682:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879298.042683:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879298.042685:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879298.042686:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879298.042687:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879298.042688:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879298.042690:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879298.042691:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879298.042693:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879298.042694:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042696:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879298.042698:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879298.042700:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879298.042702:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879298.042703:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879298.105013:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879298.105020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879298.105023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879298.105029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879298.105034:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879298.105038:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879298.105042:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879298.105045:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879298.105048:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879298.809048:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879298.809058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879298.809065:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879298.809068:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879298.809110:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879298.809113:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879298.809117:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879299.128997:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879299.129007:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879299.129010:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879299.129017:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879299.129021:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879299.129026:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879299.129029:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879299.129032:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879299.129034:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879299.832993:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879299.832999:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879299.833003:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879299.833005:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879299.833040:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879299.833044:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879299.833048:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879300.153004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879300.153011:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879300.153013:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879300.153018:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879300.153021:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879300.153025:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879300.153028:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879300.153031:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879300.153034:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879300.857046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879300.857054:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879300.857060:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879300.857062:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879300.857095:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879300.857099:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879300.857102:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879301.176996:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879301.177002:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879301.177003:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879301.177007:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879301.177010:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879301.177013:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879301.177015:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879301.177017:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879301.177019:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879301.881069:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879301.881076:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879301.881081:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879301.881082:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879301.881104:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879301.881106:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879301.881109:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879302.201000:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879302.201007:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879302.201009:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879302.201015:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879302.201018:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879302.201022:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879302.201025:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879302.201028:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879302.201030:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879302.905041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879302.905063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879302.905073:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879302.905075:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879302.905129:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879302.905135:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879302.905152:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879303.161188:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879303.161197:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879303.161201:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879303.161205:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879303.161210:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879303.161217:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10242 00000100:00080000:1.0:1788879303.161229:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879303.161232:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879303.161235:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879303.161244:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879303.161247:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879303.161252:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879303.161255:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879303.161258:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879303.161259:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879303.161264:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879303.161266:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879303.161269:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879303.161273:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879303.161277:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161280:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879303.161282:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879303.161290:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879303.161293:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161296:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879303.161299:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879303.161306:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161320:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879303.161334:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100bb80 x1875765290149760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879303.161345:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879303.161350:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161372:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879303.161377:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879303.161382:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879303.161385:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879303.161387:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879303.161392:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879303.161397:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10242 00000100:00080000:1.0:1788879303.161401:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879303.161421:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879303.161425:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879303.161431:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879303.161435:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879303.161438:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879303.161442:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879303.161445:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879303.161447:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879303.161454:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879303.161460:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879303.161463:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879303.161472:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879303.161477:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161481:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879303.161487:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879303.161492:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161495:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879303.161502:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879303.161504:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161507:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879303.161510:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879303.161514:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161520:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879303.161526:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008700 x1875765290149888/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879303.161537:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879303.161540:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161543:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879303.161547:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879303.161551:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879303.161553:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879303.161555:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879303.161559:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879303.161562:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10242 00000100:00080000:1.0:1788879303.161566:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879303.161569:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879303.161572:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879303.161573:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1788879303.161576:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879303.161583:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879303.161584:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1788879303.161586:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879303.161590:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1788879303.161593:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879303.161597:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879303.161600:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879303.161602:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161606:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:3.0:1788879303.161616:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1788879303.161617:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:3.0:1788879303.161624:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879303.161625:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879303.161635:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879303.161640:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879303.161642:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161645:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879303.161648:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879303.161652:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161654:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879303.161658:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879303.161660:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161663:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879303.161665:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879303.161668:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161680:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879303.161687:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008e00 x1875765290150016/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879303.161697:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879303.161701:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161707:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879303.161713:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879303.161718:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879303.161722:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879303.161724:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879303.161727:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879303.161730:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10242 00000100:00080000:1.0:1788879303.161734:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879303.161737:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879303.161738:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879303.161741:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879303.161743:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879303.161745:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:1.0:1788879303.161748:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879303.161751:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879303.161752:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879303.161756:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879303.161758:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879303.161760:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879303.161763:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879303.161765:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161768:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879303.161770:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879303.161772:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161774:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879303.161779:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879303.161781:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161783:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879303.161785:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879303.161787:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161793:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879303.161821:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008000 x1875765290150144/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879303.161837:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879303.161841:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879303.161862:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879303.161888:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879303.161897:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290149760:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879303.161992:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879303.162000:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879303.162003:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879303.162006:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879303.162011:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162027:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879303.162030:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286523264 : -117763423028352 : ffff94e51100bb80) 00000100:00000001:3.0:1788879303.162034:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879303.162074:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879303.162086:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879303.162093:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879303.162097:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162101:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879303.162110:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162115:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304914986 00000100:00000001:3.0:1788879303.162125:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879303.162137:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304914986 00000100:00000001:3.0:1788879303.162142:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879303.162151:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879303.162154:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162157:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879303.162166:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879303/real 1788879303] req@ffff94e51100bb80 x1875765290149760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879303.162179:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879303.162181:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879303.162184:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879303.162186:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879303.162188:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162190:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879303.162192:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879303.162198:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100bb80 x1875765290149760/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879303.162208:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162212:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879303.162215:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162218:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879303.162220:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162222:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879303.162224:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879303.162227:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879303.162231:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879303.162233:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879303.162237:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290149760:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879303.162245:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879303.162249:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879303.162253:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879303.162255:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879303.162258:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879303.162260:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879303.162264:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290149888:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879303.162268:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879303.162271:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879303.162273:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879303.162275:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879303.162277:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162282:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879303.162284:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286509824 : -117763423041792 : ffff94e511008700) 00000100:00000001:3.0:1788879303.162289:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879303.162301:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879303.162312:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879303.162320:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879303.162323:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162325:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879303.162327:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162331:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304914444 00000100:00000001:3.0:1788879303.162334:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304914986 00000100:00000001:3.0:1788879303.162336:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304914444 00000100:00000001:3.0:1788879303.162340:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304914986 00000100:00000001:3.0:1788879303.162343:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879303.162344:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162346:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879303.162350:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879303/real 1788879303] req@ffff94e511008700 x1875765290149888/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879303.162357:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879303.162358:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879303.162360:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879303.162361:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879303.162362:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162363:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879303.162365:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879303.162368:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008700 x1875765290149888/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879303.162374:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162376:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879303.162378:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162380:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879303.162381:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162382:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879303.162393:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879303.162395:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879303.162398:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879303.162400:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879303.162402:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290149888:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879303.162406:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162408:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879303.162409:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879303.162412:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879303.162414:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879303.162416:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879303.162418:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879303.162422:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879303.162423:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879303.162426:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879303.162427:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162429:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879303.162430:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879303.162431:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879303.162433:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879303.162434:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879303.162435:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879303.162436:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879303.162437:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879303.162438:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879303.162439:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162441:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162447:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879303.162449:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162451:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879303.162452:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879303.162453:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879303.162454:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879303.162455:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879303.162457:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879303.162459:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879303.162461:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879303.162463:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290150016:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879303.162466:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879303.162468:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879303.162469:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879303.162471:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879303.162473:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162477:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879303.162478:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286511616 : -117763423040000 : ffff94e511008e00) 00000100:00000001:3.0:1788879303.162482:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879303.162491:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879303.162494:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879303.162497:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879303.162498:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162499:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879303.162502:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162504:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304914442 00000100:00000001:3.0:1788879303.162507:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304914986 00000100:00000001:3.0:1788879303.162509:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304914442 00000100:00000001:3.0:1788879303.162511:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304914986 00000100:00000001:3.0:1788879303.162513:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879303.162515:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162516:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879303.162519:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879303/real 1788879303] req@ffff94e511008e00 x1875765290150016/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879303.162526:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879303.162527:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879303.162528:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879303.162530:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879303.162531:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162532:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879303.162533:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879303.162537:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008e00 x1875765290150016/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879303.162543:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162545:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879303.162547:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162548:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879303.162549:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162551:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879303.162561:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879303.162563:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879303.162566:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879303.162568:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879303.162571:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290150016:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879303.162574:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879303.162575:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879303.162577:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879303.162578:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879303.162580:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879303.162581:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879303.162583:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290150144:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879303.162586:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879303.162587:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879303.162588:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879303.162590:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879303.162592:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162595:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879303.162596:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508032 : -117763423043584 : ffff94e511008000) 00000100:00000001:3.0:1788879303.162599:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879303.162607:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879303.162610:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879303.162612:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879303.162613:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162615:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879303.162617:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162620:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304914442 00000100:00000001:3.0:1788879303.162622:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304914986 00000100:00000001:3.0:1788879303.162624:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304914442 00000100:00000001:3.0:1788879303.162626:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304914986 00000100:00000001:3.0:1788879303.162628:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879303.162629:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162631:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879303.162634:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879303/real 1788879303] req@ffff94e511008000 x1875765290150144/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879303.162647:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879303.162648:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879303.162650:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879303.162652:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879303.162653:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162654:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879303.162655:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879303.162658:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008000 x1875765290150144/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879303.162664:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162665:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879303.162667:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162669:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879303.162671:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162673:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879303.162688:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879303.162708:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879303.162713:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879303.162717:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879303.162724:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290150144:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879303.162731:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162735:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879303.162737:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879303.162740:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879303.162745:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879303.162748:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879303.162751:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879303.162755:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879303.162758:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879303.162762:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879303.162764:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162769:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879303.162772:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879303.162774:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879303.162777:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879303.162780:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879303.162782:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879303.162786:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879303.162788:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879303.162791:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879303.162794:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162827:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879303.162832:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879303.162834:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879303.162837:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879303.162839:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879303.225039:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879303.225045:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879303.225048:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879303.225054:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879303.225058:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879303.225062:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879303.225065:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879303.225068:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879303.225071:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879303.928987:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879303.928993:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879303.928997:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879303.929000:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879303.929002:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879303.929005:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879303.929007:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879303.929008:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879303.929010:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879303.929011:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879303.929074:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879303.929079:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879303.929083:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879304.249015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879304.249021:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879304.249023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879304.249026:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879304.249029:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879304.249032:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879304.249035:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879304.249037:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879304.249039:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879304.953045:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879304.953054:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879304.953060:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879304.953062:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879304.953130:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879304.953134:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879304.953139:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879305.273010:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879305.273017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879305.273019:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879305.273024:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879305.273028:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879305.273031:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879305.273034:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879305.273036:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879305.273039:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879305.977058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879305.977068:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879305.977076:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879305.977079:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879305.977121:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879305.977125:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879305.977129:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879306.297042:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879306.297051:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879306.297054:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879306.297060:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879306.297064:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879306.297069:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879306.297073:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879306.297075:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879306.297079:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879307.000997:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879307.001005:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879307.001008:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879307.001011:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879307.001015:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879307.001017:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879307.001019:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879307.001021:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879307.001024:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879307.001025:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879307.001044:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879307.001046:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879307.001049:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879307.320987:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879307.320994:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879307.320995:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879307.321000:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879307.321003:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879307.321007:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879307.321026:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879307.321028:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879307.321030:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879308.025062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879308.025069:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879308.025074:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879308.025075:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879308.025137:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879308.025141:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879308.025145:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879308.281095:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879308.281109:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879308.281113:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879308.281118:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879308.281124:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879308.281135:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10247 00000100:00080000:0.0:1788879308.281177:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879308.281183:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879308.281187:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879308.281201:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879308.281206:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879308.281212:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879308.281217:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879308.281220:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879308.281222:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879308.281228:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879308.281232:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879308.281236:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879308.281241:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879308.281245:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.281248:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879308.281251:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879308.281266:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879308.281268:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.281271:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879308.281275:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879308.281280:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.281312:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879308.281322:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290150272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879308.281332:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879308.281337:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.281374:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879308.281382:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879308.281389:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879308.281392:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879308.281395:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879308.281401:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879308.281405:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10247 00000100:00080000:0.0:1788879308.281414:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879308.281418:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879308.281420:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879308.281427:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879308.281430:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879308.281434:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879308.281438:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879308.281440:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879308.281442:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879308.281445:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879308.281447:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788879308.281451:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879308.281452:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879308.281457:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.281457:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879308.281460:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879308.281461:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879308.281467:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879308.281469:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879308.281472:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879308.281474:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.281477:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788879308.281477:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788879308.281484:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879308.281485:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290150272:192.168.203.129@tcp:250:kworker.0 00000008:00000001:0.0:1788879308.281490:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.281494:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.281494:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879308.281499:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879308.281502:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879308.281504:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879308.281504:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.281508:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879308.281508:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879308.281511:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879308.281514:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879308.281516:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.281526:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879308.281536:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979180 x1875765290150400/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879308.281539:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879308.281542:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 00000100:00000001:3.0:1788879308.281549:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879308.281563:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879308.281567:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.281573:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879308.281577:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879308.281582:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879308.281585:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879308.281589:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879308.281594:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879308.281599:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10247 00000800:00000100:3.0:1788879308.281603:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1788879308.281606:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879308.281609:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879308.281612:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879308.281618:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879308.281620:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879308.281622:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:0.0:1788879308.281624:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879308.281628:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1788879308.281629:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879308.281633:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879308.281633:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.281636:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879308.281637:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:0.0:1788879308.281642:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879308.281645:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.281647:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879308.281650:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879308.281651:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304920105 02000000:00000001:0.0:1788879308.281655:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879308.281659:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.281662:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879308.281665:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879308.281667:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00000001:0.0:1788879308.281670:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.281674:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.281677:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304920105 02000000:00000001:0.0:1788879308.281680:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879308.281682:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.281684:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788879308.281687:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879308.281689:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879308.281692:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.281701:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.281721:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788879308.281725:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290150528/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879308.281726:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.281729:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879308.281736:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879308/real 1788879308] req@ffff94e508978380 x1875765290150272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879308.281738:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879308.281741:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.281744:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879308.281748:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:3.0:1788879308.281752:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:0.0:1788879308.281753:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000001:3.0:1788879308.281755:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879308.281756:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879308.281758:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879308.281760:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879308.281781:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879308.281784:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879308.281785:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879308.281789:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10247 00000100:00000001:3.0:1788879308.281789:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879308.281792:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879308.281795:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879308.281798:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879308.281800:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:3.0:1788879308.281800:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290150272/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788879308.281805:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879308.281807:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879308.281810:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879308.281812:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788879308.281813:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879308.281816:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879308.281817:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879308.281818:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879308.281822:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879308.281823:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879308.281825:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879308.281827:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.281827:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879308.281830:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879308.281831:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879308.281834:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1788879308.281835:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879308.281838:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879308.281839:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1788879308.281841:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879308.281844:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1788879308.281845:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000008:00000001:0.0:1788879308.281848:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.281848:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1788879308.281851:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879308.281854:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290150272:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788879308.281857:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879308.281859:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.281861:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879308.281861:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879308.281863:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879308.281864:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1788879308.281866:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.281867:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879308.281871:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1788879308.281874:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879308.281875:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879308.281878:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00100000:0.0:1788879308.281880:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290150656/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:3.0:1788879308.281881:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879308.281886:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879308.281888:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879308.281891:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879308.281892:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879308.281894:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.281894:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879308.281898:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.281898:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.281963:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879308.281968:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.281974:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879308.281977:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879308.281981:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879308.281983:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879308.281986:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879308.281990:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879308.281994:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879308.281998:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879308.282005:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290150400:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879308.282012:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879308.282016:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879308.282019:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879308.282021:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879308.282026:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.282035:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879308.282036:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 00000100:00000001:3.0:1788879308.282042:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879308.282060:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879308.282066:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879308.282069:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879308.282088:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.282101:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879308.282105:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.282110:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304919564 00000100:00000001:3.0:1788879308.282113:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304920105 00000100:00000001:3.0:1788879308.282115:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304919564 00000100:00000001:3.0:1788879308.282118:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304920105 00000100:00000001:3.0:1788879308.282120:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879308.282122:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.282124:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879308.282129:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879308/real 1788879308] req@ffff94e508979180 x1875765290150400/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879308.282154:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879308.282156:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879308.282159:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879308.282161:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879308.282162:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282165:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879308.282166:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879308.282171:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290150400/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879308.282179:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282182:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879308.282184:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282186:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879308.282187:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282189:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879308.282202:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879308.282205:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879308.282209:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879308.282212:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879308.282215:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290150400:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879308.282219:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879308.282221:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879308.282222:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879308.282225:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879308.282228:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879308.282230:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879308.282234:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290150528:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879308.282238:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879308.282240:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879308.282242:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879308.282243:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879308.282247:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.282270:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879308.282272:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000100:00000001:3.0:1788879308.282275:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879308.282286:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879308.282290:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879308.282292:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879308.282294:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.282295:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879308.282297:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.282300:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304919562 00000100:00000001:3.0:1788879308.282303:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304920105 00000100:00000001:3.0:1788879308.282305:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304919562 00000100:00000001:3.0:1788879308.282307:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304920105 00000100:00000001:3.0:1788879308.282309:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879308.282310:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.282312:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879308.282332:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879308/real 1788879308] req@ffff94e50897b800 x1875765290150528/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879308.282341:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879308.282343:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879308.282345:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879308.282347:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879308.282349:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282351:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879308.282352:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879308.282357:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290150528/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879308.282366:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282368:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879308.282371:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282372:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879308.282374:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282376:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879308.282386:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879308.282389:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879308.282392:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879308.282394:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879308.282397:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290150528:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879308.282401:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879308.282403:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879308.282405:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879308.282407:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879308.282409:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879308.282412:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879308.282415:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290150656:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879308.282418:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879308.282420:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879308.282421:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879308.282423:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879308.282427:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.282431:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879308.282433:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:3.0:1788879308.282437:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879308.282448:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879308.282452:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879308.282454:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879308.282457:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.282459:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879308.282461:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.282465:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304919562 00000100:00000001:3.0:1788879308.282468:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304920105 00000100:00000001:3.0:1788879308.282470:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304919562 00000100:00000001:3.0:1788879308.282473:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304920105 00000100:00000001:3.0:1788879308.282475:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879308.282477:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.282479:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879308.282483:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879308/real 1788879308] req@ffff94e50897bb80 x1875765290150656/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879308.282491:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879308.282492:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879308.282494:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879308.282496:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879308.282497:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282499:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879308.282501:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879308.282505:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290150656/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879308.282512:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282514:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879308.282517:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282519:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879308.282521:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282522:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879308.282534:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879308.282536:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879308.282539:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879308.282541:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879308.282546:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290150656:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879308.282550:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282552:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879308.282553:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879308.282555:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879308.282558:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879308.282559:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879308.282561:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879308.282563:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879308.282565:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879308.282568:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879308.282570:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282572:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879308.282573:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879308.282575:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879308.282577:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879308.282578:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879308.282579:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879308.282581:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879308.282582:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879308.282585:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879308.282586:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282588:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879308.282589:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879308.282590:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879308.282592:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879308.282594:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879308.282595:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879308.282597:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879308.282598:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879308.282600:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879308.282601:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282603:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879308.282607:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879308.282609:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879308.282611:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879308.282613:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879308.345087:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879308.345113:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879308.345116:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879308.345122:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879308.345126:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879308.345130:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879308.345133:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879308.345136:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879308.345139:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879309.049112:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879309.049122:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879309.049130:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879309.049132:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879309.049166:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879309.049170:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879309.049174:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879309.369167:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879309.369182:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879309.369186:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879309.369195:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879309.369202:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879309.369209:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879309.369214:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879309.369220:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879309.369224:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879310.073052:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879310.073061:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879310.073069:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879310.073070:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879310.073155:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879310.073159:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879310.073164:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879310.393084:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879310.393097:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879310.393101:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879310.393110:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879310.393115:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879310.393121:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879310.393125:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879310.393129:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879310.393133:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879311.096979:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879311.097005:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879311.097012:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879311.097014:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879311.097035:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879311.097037:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879311.097039:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879311.417029:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879311.417041:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879311.417044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879311.417050:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879311.417055:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879311.417060:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879311.417064:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879311.417067:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879311.417071:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879312.120992:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879312.120998:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879312.121003:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879312.121005:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879312.121115:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879312.121121:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879312.121127:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879312.441033:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879312.441042:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879312.441044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879312.441050:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879312.441055:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879312.441060:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879312.441063:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879312.441066:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879312.441070:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879313.145053:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879313.145058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879313.145062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879313.145065:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879313.145066:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879313.145069:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879313.145072:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879313.145073:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879313.145075:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879313.145077:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879313.145094:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879313.145096:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879313.145098:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879313.400993:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879313.401001:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879313.401003:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879313.401006:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879313.401009:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879313.401015:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10252 00000100:00080000:1.0:1788879313.401024:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879313.401026:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879313.401027:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879313.401034:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879313.401036:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879313.401038:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879313.401040:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879313.401042:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879313.401043:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879313.401046:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879313.401048:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879313.401050:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879313.401053:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879313.401055:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879313.401056:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879313.401058:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879313.401112:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879313.401113:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879313.401115:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879313.401118:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879313.401122:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879313.401132:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879313.401139:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100ad80 x1875765290150784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879313.401147:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879313.401151:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879313.401162:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879313.401164:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879313.401167:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879313.401168:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879313.401170:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879313.401172:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879313.401174:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10252 00000100:00080000:1.0:1788879313.401177:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879313.401179:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879313.401181:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879313.401183:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879313.401184:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879313.401185:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879313.401188:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879313.401188:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879313.401192:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879313.401194:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879313.401195:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879313.401199:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1788879313.401200:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879313.401203:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879313.401204:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401207:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879313.401208:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879313.401213:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879313.401214:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879313.401217:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879313.401218:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879313.401223:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788879313.401224:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1788879313.401227:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879313.401231:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879313.401231:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290150784:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879313.401236:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401239:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879313.401242:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879313.401243:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879313.401245:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879313.401247:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879313.401248:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879313.401251:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879313.401252:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879313.401254:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879313.401258:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879313.401260:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879313.401266:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008000 x1875765290150912/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879313.401272:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879313.401274:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401275:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879313.401277:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401278:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286519680 : -117763423031936 : ffff94e51100ad80) 00000100:00080000:1.0:1788879313.401283:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879313.401284:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1788879313.401288:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879313.401290:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879313.401291:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879313.401293:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879313.401295:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10252 00000100:00080000:1.0:1788879313.401298:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879313.401300:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879313.401301:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879313.401303:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879313.401303:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879313.401305:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879313.401307:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879313.401309:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879313.401310:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879313.401312:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879313.401314:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879313.401315:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879313.401317:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879313.401318:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788879313.401322:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879313.401323:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879313.401326:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879313.401329:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879313.401330:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879313.401334:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879313.401336:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401336:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879313.401340:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879313.401342:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1788879313.401343:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879313.401346:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401347:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401350:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879313.401351:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401355:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879313.401357:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008e00 x1875765290151040/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879313.401359:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304925225 00000100:00000001:1.0:1788879313.401367:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879313.401368:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879313.401371:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401371:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:1.0:1788879313.401375:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879313.401377:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879313.401379:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788879313.401379:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304925225 00000100:00080000:1.0:1788879313.401382:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879313.401385:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879313.401386:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879313.401391:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10252 00000100:00000001:3.0:1788879313.401391:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1788879313.401394:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879313.401396:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879313.401397:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879313.401398:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879313.401399:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879313.401400:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:1.0:1788879313.401401:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879313.401403:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879313.401403:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879313.401406:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879313.401408:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1788879313.401410:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879313.401414:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879313.401416:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1788879313.401416:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879313/real 1788879313] req@ffff94e51100ad80 x1875765290150784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788879313.401418:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879313.401424:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879313.401428:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879313.401431:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879313.401434:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879313.401435:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1788879313.401436:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879313.401440:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1788879313.401442:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879313.401444:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:1.0:1788879313.401445:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879313.401449:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879313.401450:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879313.401450:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:1.0:1788879313.401455:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401455:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879313.401459:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879313.401464:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008700 x1875765290151168/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879313.401469:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879313.401470:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879313.401471:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401476:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879313.401478:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879313.401483:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100ad80 x1875765290150784/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879313.401489:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401492:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879313.401494:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401495:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879313.401496:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401497:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879313.401499:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879313.401500:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879313.401503:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879313.401505:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879313.401508:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290150784:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879313.401512:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401513:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879313.401514:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879313.401516:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879313.401519:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879313.401521:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879313.401522:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879313.401525:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879313.401526:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879313.401529:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879313.401530:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401531:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401536:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879313.401538:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401539:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879313.401541:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879313.401542:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879313.401543:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879313.401544:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879313.401545:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879313.401547:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879313.401549:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879313.401551:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290150912:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879313.401553:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879313.401555:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879313.401556:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879313.401557:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879313.401559:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401563:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879313.401564:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508032 : -117763423043584 : ffff94e511008000) 00000100:00000001:3.0:1788879313.401567:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879313.401576:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879313.401579:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879313.401580:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879313.401581:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401583:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879313.401584:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401587:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304924683 00000100:00000001:3.0:1788879313.401590:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304925225 00000100:00000001:3.0:1788879313.401593:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304924683 00000100:00000001:3.0:1788879313.401595:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304925225 00000100:00000001:3.0:1788879313.401597:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879313.401600:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401601:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879313.401606:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879313/real 1788879313] req@ffff94e511008000 x1875765290150912/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879313.401615:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879313.401616:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879313.401619:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879313.401621:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879313.401622:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401624:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879313.401626:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879313.401630:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008000 x1875765290150912/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879313.401639:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401641:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879313.401644:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401646:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879313.401646:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401648:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879313.401657:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879313.401659:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879313.401662:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879313.401663:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879313.401666:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290150912:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879313.401669:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879313.401670:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879313.401671:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879313.401672:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879313.401673:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879313.401675:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879313.401677:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290151040:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879313.401679:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879313.401680:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879313.401681:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879313.401683:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879313.401685:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401689:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879313.401690:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286511616 : -117763423040000 : ffff94e511008e00) 00000100:00000001:3.0:1788879313.401701:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879313.401710:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879313.401714:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879313.401715:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879313.401717:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401718:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879313.401720:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401722:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304924681 00000100:00000001:3.0:1788879313.401724:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304925225 00000100:00000001:3.0:1788879313.401726:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304924681 00000100:00000001:3.0:1788879313.401740:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304925225 00000100:00000001:3.0:1788879313.401743:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879313.401744:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401745:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879313.401748:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879313/real 1788879313] req@ffff94e511008e00 x1875765290151040/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879313.401755:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879313.401756:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879313.401758:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879313.401759:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879313.401760:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401761:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879313.401762:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879313.401764:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008e00 x1875765290151040/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879313.401768:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401770:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879313.401771:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401772:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879313.401773:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401774:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879313.401783:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879313.401784:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879313.401786:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879313.401788:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879313.401790:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290151040:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879313.401793:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879313.401794:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879313.401795:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879313.401796:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879313.401797:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879313.401799:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879313.401801:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290151168:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879313.401803:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879313.401804:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879313.401805:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879313.401806:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879313.401808:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401811:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879313.401812:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286509824 : -117763423041792 : ffff94e511008700) 00000100:00000001:3.0:1788879313.401815:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879313.401823:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879313.401825:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879313.401827:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879313.401828:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401829:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879313.401831:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401840:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304924681 00000100:00000001:3.0:1788879313.401843:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304925225 00000100:00000001:3.0:1788879313.401845:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304924681 00000100:00000001:3.0:1788879313.401847:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304925225 00000100:00000001:3.0:1788879313.401849:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879313.401851:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401853:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879313.401857:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879313/real 1788879313] req@ffff94e511008700 x1875765290151168/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879313.401874:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879313.401875:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879313.401878:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879313.401880:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879313.401881:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401883:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879313.401885:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879313.401888:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008700 x1875765290151168/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879313.401893:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401895:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879313.401896:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401897:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879313.401898:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401899:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879313.401926:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879313.401928:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879313.401930:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879313.401932:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879313.401934:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290151168:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879313.401936:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401937:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879313.401939:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879313.401941:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879313.401944:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879313.401945:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879313.401946:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879313.401949:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879313.401950:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879313.401952:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879313.401954:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401956:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879313.401957:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879313.401959:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879313.401960:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879313.401962:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879313.401963:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879313.401964:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879313.401966:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879313.401967:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879313.401969:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401970:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879313.401971:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879313.401973:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879313.401974:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879313.401976:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879313.401977:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879313.401979:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879313.401980:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879313.401981:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879313.401983:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401985:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879313.401988:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879313.401990:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879313.401992:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879313.401993:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879313.464978:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879313.464983:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879313.464985:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879313.464989:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879313.464992:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879313.464995:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879313.464997:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879313.464999:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879313.465001:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879314.168988:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879314.168994:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879314.168999:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879314.169001:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879314.169039:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879314.169053:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879314.169056:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879314.488988:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879314.488996:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879314.488998:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879314.489003:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879314.489007:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879314.489011:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879314.489014:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879314.489017:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879314.489020:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879315.193001:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879315.193008:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879315.193014:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879315.193015:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879315.193039:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879315.193042:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879315.193046:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879315.513029:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879315.513037:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879315.513039:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879315.513046:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879315.513050:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879315.513055:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879315.513058:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879315.513061:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879315.513064:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879316.217035:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879316.217043:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879316.217045:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879316.217048:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879316.217052:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879316.217054:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879316.217056:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879316.217058:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879316.217061:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879316.217062:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879316.217129:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879316.217133:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879316.217137:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879316.536993:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879316.536999:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879316.537001:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879316.537014:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879316.537018:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879316.537021:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879316.537023:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879316.537026:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879316.537028:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879317.241022:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879317.241029:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879317.241034:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879317.241035:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879317.241057:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879317.241060:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879317.241062:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879317.561012:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879317.561020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879317.561023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879317.561030:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879317.561035:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879317.561039:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879317.561043:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879317.561046:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879317.561050:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879318.265029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879318.265037:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879318.265043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879318.265044:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879318.265071:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879318.265073:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879318.265077:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879318.521094:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879318.521119:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879318.521123:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879318.521130:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879318.521137:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879318.521149:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10257 00000100:00080000:1.0:1788879318.521164:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879318.521169:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879318.521173:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879318.521185:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879318.521189:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879318.521195:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879318.521199:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879318.521203:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879318.521206:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879318.521214:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879318.521220:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879318.521225:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879318.521231:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879318.521236:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879318.521240:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879318.521244:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879318.521261:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879318.521264:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879318.521269:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879318.521273:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879318.521280:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879318.521300:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879318.521314:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100bb80 x1875765290151296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879318.521334:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879318.521340:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879318.521366:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879318.521374:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879318.521380:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879318.521383:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879318.521387:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879318.521392:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879318.521397:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10257 00000100:00080000:1.0:1788879318.521404:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879318.521408:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879318.521411:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879318.521417:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879318.521421:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879318.521424:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879318.521428:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879318.521432:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879318.521434:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879318.521441:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879318.521441:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879318.521451:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.521453:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1788879318.521456:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879318.521460:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879318.521465:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788879318.521469:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879318.521469:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879318.521481:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879318.521482:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788879318.521491:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788879318.521491:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:1.0:1788879318.521500:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788879318.521500:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1788879318.521512:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879318.521514:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290151296:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879318.521521:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.521526:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879318.521548:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879318.521549:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879318.521553:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879318.521554:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879318.521560:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879318.521561:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879318.521567:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879318.521570:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879318.521585:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.521588:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879318.521592:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286523264 : -117763423028352 : ffff94e51100bb80) 00000100:00000001:1.0:1788879318.521597:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.521598:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:1.0:1788879318.521612:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100aa00 x1875765290151424/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879318.521624:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879318.521629:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879318.521634:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879318.521638:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1788879318.521645:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:1.0:1788879318.521649:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879318.521654:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879318.521657:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879318.521660:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879318.521662:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879318.521667:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1788879318.521672:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10257 00000100:00000001:3.0:1788879318.521673:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.521681:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:1.0:1788879318.521684:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879318.521687:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879318.521691:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879318.521715:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879318.521716:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304930345 00000020:00000001:1.0:1788879318.521724:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879318.521729:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879318.521733:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879318.521735:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:1.0:1788879318.521739:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879318.521746:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879318.521748:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304930345 00000100:00000001:1.0:1788879318.521749:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879318.521760:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1788879318.521761:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879318.521770:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879318.521771:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879318.521778:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879318.521782:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879318.521784:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879318.521787:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1788879318.521789:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879318.521794:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879318.521797:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879318.521799:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879318/real 1788879318] req@ffff94e51100bb80 x1875765290151296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1788879318.521801:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879318.521807:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879318.521815:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879318.521819:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879318.521823:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:3.0:1788879318.521824:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1788879318.521825:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:3.0:1788879318.521829:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1788879318.521831:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.521836:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879318.521841:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879318.521843:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.521844:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.521853:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879318.521856:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879318.521862:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b480 x1875765290151552/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879318.521866:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100bb80 x1875765290151296/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879318.521940:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879318.521945:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879318.521949:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879318.521958:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879318.521959:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.521966:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879318.521971:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879318.521973:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879318.521977:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788879318.521978:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879318.521983:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788879318.521984:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879318.521986:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879318.521992:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879318.521995:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879318.521998:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879318.522004:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10257 00000100:00080000:3.0:1788879318.522008:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879318.522015:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:1.0:1788879318.522018:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788879318.522020:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290151296:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879318.522028:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879318.522030:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879318.522034:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879318.522035:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879318.522039:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000020:00000001:1.0:1788879318.522044:0:80852:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:3.0:1788879318.522045:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:1.0:1788879318.522051:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:3.0:1788879318.522053:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879318.522057:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879318.522058:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879318.522062:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000020:00000001:1.0:1788879318.522067:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:3.0:1788879318.522070:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1788879318.522071:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788879318.522074:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879318.522076:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879318.522080:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879318.522084:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788879318.522085:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879318.522090:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879318.522092:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879318.522097:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879318.522100:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879318.522102:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:1.0:1788879318.522103:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879318.522109:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879318.522109:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1788879318.522115:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879318.522116:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879318.522120:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00000001:1.0:1788879318.522122:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522124:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879318.522128:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522129:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879318.522134:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879318.522137:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879318.522138:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879318.522140:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879318.522142:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879318.522147:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879318.522148:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879318.522152:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879318.522157:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879318.522159:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290151424:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879318.522164:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522168:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1788879318.522175:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b800 x1875765290151680/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879318.522176:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879318.522183:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879318.522190:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788879318.522192:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879318.522197:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879318.522199:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879318.522201:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522217:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879318.522222:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:3.0:1788879318.522231:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879318.522253:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879318.522258:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879318.522262:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879318.522263:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522265:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879318.522268:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522273:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304929804 00000100:00000001:3.0:1788879318.522277:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304930345 00000100:00000001:3.0:1788879318.522279:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304929804 00000100:00000001:3.0:1788879318.522281:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304930345 00000100:00000001:3.0:1788879318.522283:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879318.522285:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522287:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879318.522292:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879318/real 1788879318] req@ffff94e51100aa00 x1875765290151424/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879318.522301:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879318.522302:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879318.522304:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879318.522306:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879318.522308:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522310:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879318.522311:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879318.522315:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290151424/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879318.522322:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522324:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879318.522326:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522327:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879318.522329:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522330:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879318.522341:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879318.522344:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879318.522348:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879318.522351:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879318.522356:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290151424:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879318.522362:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879318.522364:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879318.522366:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879318.522369:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879318.522372:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879318.522375:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879318.522380:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290151552:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879318.522385:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879318.522389:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879318.522391:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879318.522393:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879318.522398:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522407:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879318.522409:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:3.0:1788879318.522414:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879318.522433:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879318.522438:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879318.522441:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879318.522443:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522444:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879318.522447:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522451:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304929802 00000100:00000001:3.0:1788879318.522454:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304930345 00000100:00000001:3.0:1788879318.522456:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304929802 00000100:00000001:3.0:1788879318.522458:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304930345 00000100:00000001:3.0:1788879318.522460:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879318.522462:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522464:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879318.522469:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879318/real 1788879318] req@ffff94e51100b480 x1875765290151552/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879318.522477:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879318.522478:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879318.522481:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879318.522482:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879318.522484:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522486:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879318.522487:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879318.522491:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290151552/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879318.522498:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522500:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879318.522502:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522503:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879318.522504:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522506:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879318.522517:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879318.522519:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879318.522522:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879318.522524:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879318.522526:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290151552:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879318.522530:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522532:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879318.522533:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879318.522535:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879318.522537:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879318.522539:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879318.522540:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879318.522543:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879318.522544:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879318.522546:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879318.522548:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522550:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879318.522551:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879318.522552:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879318.522554:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879318.522555:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879318.522556:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879318.522557:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879318.522558:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879318.522560:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879318.522561:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522562:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522566:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879318.522567:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522569:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879318.522570:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879318.522571:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879318.522573:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879318.522574:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879318.522576:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879318.522577:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879318.522579:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879318.522582:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290151680:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879318.522585:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879318.522587:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879318.522589:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879318.522590:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879318.522593:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522597:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879318.522599:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:3.0:1788879318.522602:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879318.522613:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879318.522616:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879318.522618:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879318.522619:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522621:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879318.522624:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522627:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304929802 00000100:00000001:3.0:1788879318.522630:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304930345 00000100:00000001:3.0:1788879318.522632:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304929802 00000100:00000001:3.0:1788879318.522634:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304930345 00000100:00000001:3.0:1788879318.522636:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879318.522638:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522639:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879318.522643:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879318/real 1788879318] req@ffff94e51100b800 x1875765290151680/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879318.522650:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879318.522651:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879318.522653:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879318.522654:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879318.522656:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522657:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879318.522659:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879318.522662:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290151680/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879318.522668:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522670:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879318.522672:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522674:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879318.522675:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522676:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879318.522686:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879318.522689:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879318.522716:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879318.522720:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879318.522724:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290151680:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879318.522730:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522734:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879318.522736:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879318.522738:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879318.522742:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879318.522744:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879318.522746:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879318.522749:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879318.522751:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879318.522753:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879318.522755:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522758:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879318.522763:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879318.522766:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879318.522769:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879318.522771:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879318.585016:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879318.585024:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879318.585026:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879318.585032:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879318.585036:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879318.585040:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879318.585043:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879318.585046:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879318.585049:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879319.288982:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879319.288988:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879319.288992:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879319.288994:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879319.289021:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879319.289023:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879319.289026:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879319.609035:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879319.609042:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879319.609044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879319.609049:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879319.609052:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879319.609055:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879319.609076:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879319.609079:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879319.609083:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879320.313041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879320.313048:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879320.313053:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879320.313055:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879320.313138:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879320.313145:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879320.313150:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879320.633009:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879320.633020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879320.633023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879320.633033:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879320.633040:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879320.633046:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879320.633051:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879320.633055:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879320.633060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879321.337001:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879321.337010:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879321.337016:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879321.337018:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879321.337047:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879321.337052:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879321.337058:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879321.656996:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879321.657004:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879321.657008:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879321.657014:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879321.657018:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879321.657023:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879321.657026:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879321.657029:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879321.657032:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879322.361056:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879322.361067:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879322.361077:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879322.361080:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879322.361113:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879322.361117:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879322.361120:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879322.681044:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879322.681055:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879322.681059:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879322.681069:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879322.681076:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879322.681082:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879322.681087:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879322.681092:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879322.681096:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879323.385045:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879323.385053:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879323.385059:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879323.385064:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879323.385067:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879323.385072:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879323.385075:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879323.385078:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879323.385097:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879323.385101:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879323.385125:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879323.385128:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879323.385131:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879323.641044:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879323.641062:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879323.641066:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879323.641073:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879323.641081:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879323.641093:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10262 00000100:00080000:1.0:1788879323.641111:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879323.641117:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879323.641120:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879323.641134:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879323.641139:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879323.641145:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879323.641151:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879323.641155:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879323.641158:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879323.641167:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879323.641171:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879323.641175:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879323.641179:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879323.641183:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879323.641187:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879323.641190:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879323.641204:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879323.641206:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879323.641210:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879323.641214:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879323.641220:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879323.641237:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879323.641249:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100a680 x1875765290151808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879323.641272:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879323.641278:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879323.641301:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879323.641306:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879323.641311:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879323.641313:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879323.641315:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879323.641319:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879323.641322:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10262 00000100:00080000:1.0:1788879323.641327:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879323.641330:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879323.641332:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879323.641337:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879323.641339:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879323.641343:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879323.641346:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879323.641350:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879323.641353:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879323.641358:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879323.641361:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879323.641364:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879323.641372:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879323.641374:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1788879323.641376:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879323.641383:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879323.641384:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00100000:1.0:1788879323.641388:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879323.641390:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879323.641396:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:1.0:1788879323.641398:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.641402:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879323.641404:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879323.641415:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879323.641416:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879323.641423:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879323.641424:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879323.641429:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879323.641433:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879323.641434:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879323.641442:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879323.641448:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290151808:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879323.641452:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.641464:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1788879323.641469:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009f80 x1875765290151936/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879323.641472:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879323.641478:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879323.641487:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788879323.641496:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788879323.641501:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879323.641502:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879323.641512:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879323.641517:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879323.641521:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879323.641523:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879323.641525:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879323.641529:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879323.641533:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10262 00000100:00000001:3.0:1788879323.641534:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1788879323.641538:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879323.641543:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286517888 : -117763423033728 : ffff94e51100a680) 00000100:00000001:1.0:1788879323.641544:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879323.641548:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879323.641556:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879323.641559:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879323.641559:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1788879323.641567:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879323.641570:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879323.641573:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879323.641576:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879323.641580:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879323.641583:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879323.641586:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879323.641591:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879323.641593:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879323.641595:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879323.641598:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879323.641603:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879323.641605:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879323.641612:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879323.641614:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879323.641616:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879323.641618:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879323.641620:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879323.641628:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879323.641633:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009180 x1875765290152064/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879323.641647:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000100:3.0:1788879323.641649:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879323.641651:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879323.641684:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879323.641692:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879323.641696:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:1.0:1788879323.641697:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879323.641706:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879323.641709:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879323.641712:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879323.641715:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879323.641723:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879323.641725:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10262 00000100:00000001:3.0:1788879323.641735:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:1.0:1788879323.641738:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879323.641745:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879323.641747:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879323.641754:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879323.641755:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879323.641759:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879323.641770:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304935465 00000020:00000001:1.0:1788879323.641775:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:1.0:1788879323.641781:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879323.641784:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879323.641786:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879323.641791:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879323.641794:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879323.641798:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879323.641801:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1788879323.641803:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879323.641812:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879323.641814:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879323.641818:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879323.641820:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304935465 00000008:00000001:1.0:1788879323.641824:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879323.641833:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.641837:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1788879323.641844:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879323.641849:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879323.641852:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.641860:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879323.641862:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879323.641869:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879323.641876:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.641887:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1788879323.641888:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879323.641947:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879323/real 1788879323] req@ffff94e51100a680 x1875765290151808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879323.641958:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008380 x1875765290152192/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788879323.641980:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1788879323.641990:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:3.0:1788879323.641997:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879323.642000:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.642007:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879323.642011:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.642015:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879323.642021:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642025:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879323.642028:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879323.642037:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100a680 x1875765290151808/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879323.642049:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642055:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879323.642059:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642061:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879323.642063:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642066:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879323.642068:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879323.642072:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879323.642078:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879323.642081:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879323.642086:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290151808:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879323.642093:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642096:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879323.642099:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879323.642102:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879323.642106:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879323.642109:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879323.642113:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879323.642119:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879323.642122:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879323.642128:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879323.642131:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642136:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642146:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879323.642150:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.642154:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879323.642156:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879323.642159:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879323.642162:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879323.642165:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879323.642170:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879323.642174:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879323.642179:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879323.642186:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290151936:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879323.642195:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879323.642199:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879323.642202:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879323.642206:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879323.642212:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.642224:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879323.642226:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286516096 : -117763423035520 : ffff94e511009f80) 00000100:00000001:3.0:1788879323.642232:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879323.642254:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879323.642261:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879323.642265:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879323.642267:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.642269:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879323.642273:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.642278:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304934924 00000100:00000001:3.0:1788879323.642282:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304935465 00000100:00000001:3.0:1788879323.642285:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304934924 00000100:00000001:3.0:1788879323.642289:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304935465 00000100:00000001:3.0:1788879323.642292:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879323.642295:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.642297:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879323.642303:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879323/real 1788879323] req@ffff94e511009f80 x1875765290151936/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879323.642315:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879323.642317:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879323.642320:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879323.642322:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879323.642324:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642326:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879323.642328:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879323.642333:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009f80 x1875765290151936/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879323.642342:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642345:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879323.642348:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642350:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879323.642351:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642353:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879323.642368:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879323.642371:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879323.642374:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879323.642377:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879323.642381:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290151936:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879323.642385:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879323.642387:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879323.642389:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879323.642392:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879323.642395:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879323.642398:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879323.642402:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290152064:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879323.642406:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879323.642408:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879323.642410:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879323.642412:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879323.642416:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.642422:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879323.642424:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286512512 : -117763423039104 : ffff94e511009180) 00000100:00000001:3.0:1788879323.642428:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879323.642440:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879323.642445:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879323.642448:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879323.642450:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.642452:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879323.642455:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.642459:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304934922 00000100:00000001:3.0:1788879323.642462:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304935465 00000100:00000001:3.0:1788879323.642465:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304934922 00000100:00000001:3.0:1788879323.642467:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304935465 00000100:00000001:3.0:1788879323.642470:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879323.642472:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.642474:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879323.642479:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879323/real 1788879323] req@ffff94e511009180 x1875765290152064/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879323.642489:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879323.642490:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879323.642492:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879323.642494:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879323.642496:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642499:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879323.642501:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879323.642510:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009180 x1875765290152064/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879323.642524:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642528:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879323.642532:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642536:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879323.642538:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642542:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879323.642562:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879323.642569:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879323.642576:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879323.642581:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879323.642588:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290152064:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879323.642595:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879323.642598:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879323.642601:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879323.642605:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879323.642607:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879323.642610:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879323.642614:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290152192:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879323.642618:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879323.642621:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879323.642623:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879323.642625:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879323.642630:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.642637:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879323.642639:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508928 : -117763423042688 : ffff94e511008380) 00000100:00000001:3.0:1788879323.642644:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879323.642691:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879323.642701:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879323.642706:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879323.642710:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.642713:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879323.642719:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.642728:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304934922 00000100:00000001:3.0:1788879323.642733:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304935465 00000100:00000001:3.0:1788879323.642738:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304934922 00000100:00000001:3.0:1788879323.642743:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304935465 00000100:00000001:3.0:1788879323.642747:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879323.642751:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.642756:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879323.642765:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879323/real 1788879323] req@ffff94e511008380 x1875765290152192/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879323.642779:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879323.642781:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879323.642785:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879323.642787:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879323.642789:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642792:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879323.642794:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879323.642799:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008380 x1875765290152192/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879323.642814:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642818:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879323.642823:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642826:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879323.642828:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642832:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879323.642853:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879323.642859:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879323.642867:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879323.642871:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879323.642879:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290152192:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879323.642887:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642891:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879323.642894:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879323.642897:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879323.642931:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879323.642934:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879323.642936:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879323.642940:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879323.642942:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879323.642945:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879323.642947:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642949:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879323.642951:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879323.642953:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879323.642955:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879323.642957:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879323.642958:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879323.642960:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879323.642962:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879323.642964:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879323.642965:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642967:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879323.642968:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879323.642970:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879323.642972:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879323.642973:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879323.642974:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879323.642976:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879323.642977:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879323.642979:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879323.642981:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642983:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879323.642988:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879323.642990:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879323.642992:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879323.642995:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879323.705010:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879323.705017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879323.705020:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879323.705026:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879323.705030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879323.705034:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879323.705038:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879323.705041:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879323.705044:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879324.409039:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879324.409047:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879324.409057:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879324.409060:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879324.409208:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879324.409214:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879324.409219:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879324.729026:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879324.729034:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879324.729036:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879324.729043:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879324.729047:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879324.729052:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879324.729056:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879324.729059:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879324.729062:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879325.433041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879325.433052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879325.433062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879325.433065:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879325.433098:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879325.433101:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879325.433104:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879325.753017:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879325.753026:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879325.753029:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879325.753035:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879325.753040:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879325.753044:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879325.753048:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879325.753051:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879325.753055:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879326.457153:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879326.457169:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879326.457175:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879326.457182:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879326.457190:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879326.457194:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879326.457198:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879326.457202:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879326.457209:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879326.457212:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879326.457270:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879326.457275:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879326.457279:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879326.777062:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879326.777074:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879326.777077:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879326.777086:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879326.777092:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879326.777097:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879326.777102:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879326.777106:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879326.777109:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879327.481157:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879327.481171:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879327.481183:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879327.481186:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879327.481231:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879327.481238:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879327.481244:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879327.801029:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879327.801039:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879327.801043:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879327.801053:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879327.801058:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879327.801065:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879327.801070:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879327.801073:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879327.801076:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879328.505019:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879328.505025:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879328.505031:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879328.505033:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879328.505139:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879328.505144:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879328.505149:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879328.761023:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879328.761033:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879328.761036:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879328.761039:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879328.761043:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879328.761051:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10267 00000100:00080000:3.0:1788879328.761061:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879328.761064:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879328.761066:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879328.761073:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879328.761075:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879328.761078:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879328.761080:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879328.761082:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879328.761083:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879328.761087:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879328.761089:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879328.761092:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879328.761094:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879328.761097:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879328.761098:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879328.761100:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879328.761110:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879328.761111:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879328.761114:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879328.761116:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879328.761120:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879328.761131:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879328.761139:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290152320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879328.761149:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879328.761153:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879328.761170:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879328.761174:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879328.761179:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879328.761180:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879328.761181:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879328.761184:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879328.761187:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10267 00000100:00080000:3.0:1788879328.761190:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879328.761192:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879328.761193:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879328.761196:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879328.761197:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879328.761199:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879328.761201:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879328.761203:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879328.761205:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879328.761205:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879328.761211:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788879328.761212:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879328.761217:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879328.761217:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761222:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788879328.761222:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879328.761226:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879328.761228:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879328.761231:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879328.761233:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879328.761235:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879328.761236:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879328.761242:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788879328.761243:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879328.761250:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290152320:192.168.203.129@tcp:250:kworker.0 00000008:00000001:3.0:1788879328.761250:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879328.761255:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761259:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879328.761262:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879328.761264:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879328.761265:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879328.761269:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879328.761270:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879328.761275:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879328.761276:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879328.761279:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879328.761283:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879328.761287:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879328.761295:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290152448/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879328.761300:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879328.761304:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:1.0:1788879328.761310:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879328.761315:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879328.761317:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879328.761321:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879328.761323:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879328.761326:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879328.761327:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879328.761329:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879328.761331:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879328.761333:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10267 00000100:00080000:3.0:1788879328.761337:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879328.761339:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879328.761340:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879328.761342:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879328.761343:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879328.761345:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000800:00000100:1.0:1788879328.761349:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:3.0:1788879328.761350:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879328.761355:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879328.761357:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879328.761361:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879328.761363:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879328.761364:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879328.761368:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879328.761371:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788879328.761372:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879328.761375:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879328.761377:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761378:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879328.761381:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879328.761386:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788879328.761386:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879328.761392:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761393:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304940585 00000100:00000001:3.0:1788879328.761394:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879328.761402:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879328.761404:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879328.761406:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879328.761407:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788879328.761408:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879328.761412:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761417:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304940585 00000100:00000001:3.0:1788879328.761418:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761426:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00100000:3.0:1788879328.761427:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290152576/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879328.761433:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879328.761439:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761444:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879328.761445:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879328.761449:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879328.761453:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879328/real 1788879328] req@ffff94e50449c380 x1875765290152320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879328.761454:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879328.761460:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879328.761470:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879328.761474:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879328.761478:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:1.0:1788879328.761480:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788879328.761482:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000400:00000001:1.0:1788879328.761483:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1788879328.761488:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10267 00000100:00000001:1.0:1788879328.761489:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:3.0:1788879328.761492:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879328.761493:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879328.761496:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879328.761497:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879328.761499:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879328.761502:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1788879328.761504:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879328.761505:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788879328.761507:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879328.761510:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:1.0:1788879328.761514:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290152320/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788879328.761515:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879328.761521:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879328.761524:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879328.761534:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788879328.761535:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879328.761541:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879328.761542:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879328.761547:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879328.761549:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879328.761552:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879328.761553:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788879328.761556:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761557:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879328.761561:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879328.761562:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000008:00100000:3.0:1788879328.761565:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879328.761566:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:3.0:1788879328.761574:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879328.761575:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788879328.761579:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879328.761582:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879328.761584:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879328.761587:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290152320:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879328.761596:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879328.761598:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879328.761599:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788879328.761602:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761603:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1788879328.761608:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879328.761609:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879328.761612:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879328.761614:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:3.0:1788879328.761616:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879328.761618:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:3.0:1788879328.761635:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879328.761639:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00100000:3.0:1788879328.761644:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290152704/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879328.761647:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879328.761652:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879328.761657:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879328.761658:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879328.761660:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761662:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879328.761664:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761667:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.761676:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879328.761678:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761680:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879328.761682:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879328.761684:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879328.761685:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879328.761686:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879328.761689:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879328.761691:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879328.761693:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879328.761697:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290152448:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879328.761700:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879328.761702:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879328.761704:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879328.761705:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879328.761708:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761714:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879328.761715:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:1.0:1788879328.761720:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879328.761739:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879328.761744:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879328.761746:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879328.761747:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761748:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879328.761751:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761754:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304940043 00000100:00000001:1.0:1788879328.761757:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304940585 00000100:00000001:1.0:1788879328.761759:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304940043 00000100:00000001:1.0:1788879328.761761:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304940585 00000100:00000001:1.0:1788879328.761763:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879328.761764:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761765:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879328.761769:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879328/real 1788879328] req@ffff94e50449fb80 x1875765290152448/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879328.761775:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879328.761777:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879328.761779:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879328.761780:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879328.761782:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.761783:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879328.761784:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879328.761787:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290152448/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879328.761793:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.761795:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879328.761797:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.761798:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879328.761800:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.761801:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879328.761812:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879328.761814:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879328.761817:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879328.761819:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879328.761822:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290152448:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879328.761825:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879328.761826:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879328.761827:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879328.761828:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879328.761829:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879328.761831:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879328.761833:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290152576:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879328.761836:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879328.761837:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879328.761838:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879328.761840:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879328.761842:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761846:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879328.761847:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:1.0:1788879328.761849:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879328.761857:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879328.761860:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879328.761862:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879328.761863:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761864:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879328.761866:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761868:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304940041 00000100:00000001:1.0:1788879328.761871:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304940585 00000100:00000001:1.0:1788879328.761872:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304940041 00000100:00000001:1.0:1788879328.761874:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304940585 00000100:00000001:1.0:1788879328.761876:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879328.761878:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.761879:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879328.761882:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879328/real 1788879328] req@ffff94e50449d880 x1875765290152576/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879328.761889:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879328.761889:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879328.761891:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879328.761892:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879328.761893:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.761894:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879328.761896:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879328.761898:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290152576/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879328.761926:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.761929:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879328.761932:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.761934:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879328.761936:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.761938:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879328.761952:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879328.761955:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879328.761961:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879328.761964:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879328.761969:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290152576:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879328.761974:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879328.761976:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879328.761978:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879328.761980:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879328.761983:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879328.761986:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879328.761990:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290152704:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879328.761994:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879328.761997:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879328.761998:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879328.762000:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879328.762003:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.762008:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879328.762009:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:1.0:1788879328.762012:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879328.762024:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879328.762028:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879328.762030:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879328.762031:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.762032:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879328.762034:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.762037:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304940042 00000100:00000001:1.0:1788879328.762039:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304940585 00000100:00000001:1.0:1788879328.762041:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304940042 00000100:00000001:1.0:1788879328.762043:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304940585 00000100:00000001:1.0:1788879328.762044:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879328.762046:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.762047:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879328.762051:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879328/real 1788879328] req@ffff94e50449f800 x1875765290152704/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879328.762056:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879328.762057:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879328.762059:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879328.762060:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879328.762062:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.762063:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879328.762064:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879328.762067:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290152704/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879328.762078:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.762080:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879328.762082:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.762083:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879328.762084:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.762086:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879328.762096:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879328.762098:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879328.762100:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879328.762101:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879328.762103:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290152704:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879328.762107:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.762108:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879328.762109:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879328.762111:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879328.762113:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879328.762114:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879328.762115:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879328.762117:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879328.762118:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879328.762120:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879328.762121:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.762123:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879328.762124:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879328.762125:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879328.762126:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879328.762127:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879328.762128:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879328.762129:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879328.762130:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879328.762131:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879328.762132:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.762133:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879328.762134:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879328.762136:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879328.762137:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879328.762138:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879328.762139:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879328.762140:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879328.762141:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879328.762142:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879328.762144:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.762145:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879328.762148:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879328.762149:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879328.762150:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879328.762151:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879328.824980:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879328.824984:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879328.824986:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879328.824989:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879328.824991:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879328.824993:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879328.824994:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879328.824996:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879328.824998:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879329.529016:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879329.529023:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879329.529029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879329.529031:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879329.529055:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879329.529057:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879329.529060:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879329.849089:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879329.849097:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879329.849101:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879329.849107:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879329.849112:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879329.849117:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879329.849120:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879329.849123:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879329.849126:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879330.553000:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879330.553008:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879330.553014:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879330.553016:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879330.553039:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879330.553042:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879330.553046:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879330.873002:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879330.873010:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879330.873013:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879330.873019:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879330.873024:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879330.873028:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879330.873032:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879330.873035:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879330.873038:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879331.577036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879331.577046:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879331.577055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879331.577058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879331.577095:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879331.577101:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879331.577106:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879331.897021:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879331.897028:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879331.897031:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879331.897037:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879331.897041:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879331.897046:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879331.897049:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879331.897052:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879331.897056:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879332.600979:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879332.600984:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879332.600988:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879332.600990:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879332.601047:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879332.601051:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879332.601055:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879332.921026:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879332.921032:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879332.921035:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879332.921040:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879332.921044:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879332.921047:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879332.921050:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879332.921052:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879332.921055:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879333.625023:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879333.625031:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879333.625037:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879333.625041:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879333.625045:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879333.625050:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879333.625053:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879333.625056:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879333.625059:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879333.625062:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879333.625086:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879333.625089:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879333.625093:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879333.881039:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879333.881052:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879333.881056:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879333.881061:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879333.881067:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879333.881076:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10272 00000100:00080000:2.0:1788879333.881089:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879333.881093:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879333.881095:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879333.881105:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879333.881108:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879333.881124:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879333.881127:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879333.881130:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879333.881132:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879333.881138:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879333.881141:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879333.881145:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879333.881149:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879333.881153:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879333.881156:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879333.881158:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879333.881172:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879333.881174:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879333.881178:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879333.881192:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879333.881198:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879333.881211:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879333.881224:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290152832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879333.881238:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879333.881244:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879333.881266:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879333.881270:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879333.881275:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879333.881277:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879333.881279:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879333.881293:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879333.881296:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10272 00000100:00080000:2.0:1788879333.881301:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879333.881303:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879333.881305:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879333.881310:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879333.881311:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879333.881314:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879333.881318:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879333.881322:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879333.881324:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879333.881328:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879333.881331:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879333.881333:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879333.881339:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879333.881341:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879333.881348:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879333.881358:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879333.881359:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1788879333.881364:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879333.881367:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879333.881374:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:2.0:1788879333.881375:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.881380:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879333.881382:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879333.881392:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879333.881394:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879333.881398:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879333.881400:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879333.881408:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879333.881409:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879333.881416:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879333.881422:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879333.881433:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290152832:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879333.881437:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.881444:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879333.881452:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:2.0:1788879333.881453:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290152960/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879333.881456:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879333.881466:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879333.881475:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879333.881477:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879333.881485:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879333.881490:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879333.881496:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879333.881499:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879333.881502:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879333.881504:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879333.881508:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879333.881512:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10272 00000100:00000001:1.0:1788879333.881514:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788879333.881517:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879333.881519:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 00000100:00000001:2.0:1788879333.881522:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879333.881529:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879333.881530:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1788879333.881536:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879333.881540:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879333.881543:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879333.881547:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879333.881565:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879333.881568:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879333.881573:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879333.881577:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879333.881585:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:2.0:1788879333.881586:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879333.881620:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879333.881624:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879333.881627:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879333.881632:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879333.881633:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00000001:2.0:1788879333.881651:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.881652:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879333.881658:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.881660:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.881666:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:2.0:1788879333.881670:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879333.881676:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879333.881676:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879333.881685:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879333.881689:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879333.881690:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304945705 02000000:00000001:2.0:1788879333.881693:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.881706:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788879333.881707:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.881721:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304945705 00000100:00100000:2.0:1788879333.881722:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290153088/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879333.881734:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879333.881742:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879333.881744:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879333.881749:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879333.881751:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.881755:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879333.881759:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879333.881765:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788879333.881771:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879333/real 1788879333] req@ffff94e5045e5180 x1875765290152832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879333.881772:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879333.881779:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879333.881784:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879333.881796:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000400:00000001:1.0:1788879333.881807:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1788879333.881809:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10272 00000400:00000001:1.0:1788879333.881811:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879333.881818:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:2.0:1788879333.881825:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879333.881826:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879333.881831:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.881838:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879333.881838:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879333.881844:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879333.881848:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:1.0:1788879333.881855:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290152832/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788879333.881861:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879333.881868:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879333.881871:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788879333.881876:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879333.881879:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879333.881888:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788879333.881890:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879333.881892:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879333.881896:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.881944:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879333.881947:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879333.881952:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879333.881959:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879333.881961:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879333.881964:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788879333.881967:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:1.0:1788879333.881976:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290152832:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1788879333.881982:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879333.881991:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879333.881992:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882000:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788879333.882001:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879333.882006:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879333.882009:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879333.882012:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879333.882020:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879333.882021:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1788879333.882024:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000020:00000001:1.0:1788879333.882027:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000008:00100000:2.0:1788879333.882031:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000020:00000001:1.0:1788879333.882032:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000008:00000001:2.0:1788879333.882040:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879333.882044:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1788879333.882045:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879333.882050:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1788879333.882056:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879333.882057:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879333.882064:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879333.882065:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882070:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879333.882072:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879333.882079:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879333.882082:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882085:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879333.882090:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879333.882091:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882098:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1788879333.882104:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290153216/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879333.882116:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879333.882121:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879333.882125:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879333.882130:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879333.882133:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879333.882136:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879333.882142:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879333.882143:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879333.882150:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879333.882150:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879333.882160:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290152960:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879333.882165:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879333.882168:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879333.882171:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879333.882174:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879333.882182:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882197:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879333.882201:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:1.0:1788879333.882209:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879333.882236:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879333.882244:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879333.882250:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879333.882253:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882257:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879333.882263:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882271:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304945164 00000100:00000001:1.0:1788879333.882277:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304945705 00000100:00000001:1.0:1788879333.882282:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304945164 00000100:00000001:1.0:1788879333.882287:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304945705 00000100:00000001:1.0:1788879333.882301:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879333.882303:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882306:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879333.882312:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879333/real 1788879333] req@ffff94e5045e5880 x1875765290152960/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879333.882324:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879333.882326:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879333.882329:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879333.882331:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879333.882333:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882336:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879333.882337:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879333.882342:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290152960/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879333.882350:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882353:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879333.882356:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882358:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879333.882360:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882362:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879333.882378:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879333.882382:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879333.882386:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879333.882389:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879333.882393:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290152960:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879333.882397:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879333.882399:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879333.882401:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879333.882404:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879333.882407:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879333.882409:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879333.882413:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290153088:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879333.882416:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879333.882419:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879333.882421:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879333.882423:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879333.882434:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882441:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879333.882442:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:1.0:1788879333.882447:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879333.882463:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879333.882468:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879333.882471:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879333.882473:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882475:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879333.882478:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882482:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304945162 00000100:00000001:1.0:1788879333.882485:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304945705 00000100:00000001:1.0:1788879333.882498:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304945162 00000100:00000001:1.0:1788879333.882501:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304945705 00000100:00000001:1.0:1788879333.882503:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879333.882506:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882508:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879333.882512:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879333/real 1788879333] req@ffff94e5045e6a00 x1875765290153088/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879333.882521:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879333.882522:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879333.882525:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879333.882526:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879333.882528:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882531:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879333.882533:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879333.882537:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290153088/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879333.882546:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882548:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879333.882551:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882553:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879333.882554:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882556:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879333.882568:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879333.882571:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879333.882574:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879333.882577:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879333.882580:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290153088:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879333.882584:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882614:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879333.882617:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879333.882619:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879333.882622:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879333.882624:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879333.882636:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879333.882639:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879333.882641:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879333.882644:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879333.882645:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882648:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879333.882650:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879333.882651:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879333.882654:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879333.882655:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879333.882657:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879333.882660:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879333.882661:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879333.882663:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879333.882666:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882668:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882673:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879333.882675:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882677:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879333.882679:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879333.882681:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879333.882683:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879333.882685:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879333.882688:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879333.882690:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879333.882692:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879333.882696:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290153216:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879333.882700:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879333.882703:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879333.882705:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879333.882707:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879333.882711:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882718:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879333.882720:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:1.0:1788879333.882725:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879333.882740:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879333.882745:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879333.882748:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879333.882749:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882752:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879333.882755:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882759:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304945162 00000100:00000001:1.0:1788879333.882762:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304945705 00000100:00000001:1.0:1788879333.882765:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304945162 00000100:00000001:1.0:1788879333.882768:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304945705 00000100:00000001:1.0:1788879333.882770:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879333.882772:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882774:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879333.882779:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879333/real 1788879333] req@ffff94e5045e6680 x1875765290153216/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879333.882789:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879333.882791:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879333.882794:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879333.882796:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879333.882797:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882800:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879333.882801:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879333.882806:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290153216/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879333.882814:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882817:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879333.882819:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882821:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879333.882823:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882825:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879333.882837:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879333.882841:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879333.882845:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879333.882848:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879333.882852:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290153216:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879333.882856:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882858:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879333.882859:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879333.882861:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879333.882863:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879333.882864:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879333.882865:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879333.882868:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879333.882869:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879333.882871:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879333.882873:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882876:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879333.882879:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879333.882881:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879333.882883:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879333.882885:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879333.945010:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879333.945016:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879333.945018:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879333.945023:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879333.945027:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879333.945030:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879333.945033:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879333.945036:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879333.945039:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879334.649024:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879334.649032:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879334.649041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879334.649044:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879334.649077:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879334.649082:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879334.649087:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879334.969035:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879334.969042:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879334.969043:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879334.969047:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879334.969050:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879334.969053:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879334.969055:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879334.969058:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879334.969060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879335.672996:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879335.673004:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879335.673010:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879335.673012:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879335.673037:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879335.673042:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879335.673047:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879335.993046:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879335.993055:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879335.993058:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879335.993065:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879335.993069:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879335.993074:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879335.993078:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879335.993081:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879335.993084:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879336.697016:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879336.697025:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879336.697029:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879336.697033:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879336.697039:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879336.697042:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879336.697045:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879336.697048:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879336.697053:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879336.697055:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879336.697131:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879336.697137:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879336.697142:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879337.016995:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879337.017003:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879337.017006:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879337.017012:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879337.017017:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879337.017021:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879337.017024:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879337.017027:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879337.017030:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879337.721029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879337.721039:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879337.721047:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879337.721049:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879337.721079:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879337.721083:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879337.721086:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879338.040986:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879338.040996:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879338.041000:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879338.041008:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879338.041013:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879338.041018:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879338.041023:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879338.041027:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879338.041031:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879338.745039:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879338.745063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879338.745070:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879338.745072:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879338.745118:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879338.745120:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879338.745123:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879339.001026:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879339.001063:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879339.001065:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879339.001070:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879339.001074:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879339.001082:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10277 00000100:00080000:2.0:1788879339.001093:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879339.001096:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879339.001098:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879339.001107:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879339.001111:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879339.001115:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879339.001117:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879339.001120:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879339.001123:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879339.001131:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879339.001159:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879339.001164:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879339.001170:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879339.001176:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001180:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879339.001185:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879339.001199:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879339.001203:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001208:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879339.001211:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879339.001229:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001243:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879339.001255:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290153344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879339.001267:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879339.001272:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001306:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879339.001311:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879339.001316:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879339.001318:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879339.001321:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879339.001325:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879339.001330:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10277 00000100:00080000:2.0:1788879339.001338:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879339.001341:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879339.001343:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879339.001345:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788879339.001350:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879339.001352:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788879339.001355:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879339.001358:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788879339.001360:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879339.001363:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1788879339.001365:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879339.001368:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001370:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879339.001376:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:3.0:1788879339.001377:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879339.001384:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788879339.001386:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:3.0:1788879339.001389:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001393:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879339.001400:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290153344:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879339.001402:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879339.001410:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879339.001415:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879339.001423:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879339.001426:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788879339.001427:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788879339.001431:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788879339.001437:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788879339.001437:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879339.001453:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788879339.001456:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001463:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879339.001472:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879339.001473:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788879339.001475:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.001477:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:2.0:1788879339.001484:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879339.001488:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879339.001492:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001499:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879339.001506:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290153472/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879339.001515:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879339.001517:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001521:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879339.001524:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879339.001527:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879339.001529:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879339.001531:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879339.001533:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879339.001536:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10277 00000100:00080000:2.0:1788879339.001540:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879339.001543:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879339.001544:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879339.001549:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879339.001573:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879339.001587:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879339.001590:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879339.001592:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879339.001594:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879339.001599:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879339.001620:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879339.001624:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879339.001626:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1788879339.001630:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879339.001636:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001639:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879339.001643:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879339.001647:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001650:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879339.001658:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879339.001660:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001663:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879339.001665:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879339.001669:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001678:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879339.001704:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290153600/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879339.001713:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879339.001715:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001718:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879339.001720:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879339.001724:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879339.001725:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879339.001727:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879339.001730:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879339.001732:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10277 00000100:00080000:2.0:1788879339.001737:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879339.001739:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879339.001741:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879339.001744:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879339.001746:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879339.001749:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879339.001751:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879339.001753:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879339.001754:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879339.001757:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879339.001760:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879339.001770:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879339.001773:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879339.001775:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001777:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879339.001779:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879339.001781:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001783:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879339.001789:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879339.001790:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001809:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879339.001810:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879339.001813:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001820:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879339.001827:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290153728/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879339.001836:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879339.001839:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879339.001841:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788879339.002017:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879339.002057:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879339.002064:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879339.002069:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002072:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879339.002082:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002088:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304950826 00000100:00000001:3.0:1788879339.002101:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879339.002110:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304950826 00000100:00000001:3.0:1788879339.002117:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879339.002134:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879339.002137:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002141:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879339.002148:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879339/real 1788879339] req@ffff94e5045e4700 x1875765290153344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879339.002161:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879339.002163:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879339.002168:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879339.002170:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879339.002172:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002175:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879339.002177:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879339.002181:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290153344/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879339.002189:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002193:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879339.002196:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002198:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879339.002199:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002201:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879339.002203:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879339.002206:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879339.002210:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879339.002213:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879339.002217:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290153344:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879339.002222:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002225:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879339.002227:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879339.002229:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879339.002233:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879339.002235:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879339.002237:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879339.002242:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879339.002243:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879339.002246:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879339.002247:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002250:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002267:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879339.002269:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002271:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879339.002273:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879339.002274:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879339.002276:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879339.002278:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879339.002280:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879339.002282:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879339.002285:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879339.002288:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290153472:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879339.002291:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879339.002294:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879339.002296:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879339.002298:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879339.002301:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002307:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879339.002308:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00000001:3.0:1788879339.002313:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879339.002330:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879339.002334:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879339.002337:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879339.002338:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002340:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879339.002343:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002348:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304950284 00000100:00000001:3.0:1788879339.002351:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304950826 00000100:00000001:3.0:1788879339.002354:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304950284 00000100:00000001:3.0:1788879339.002356:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304950826 00000100:00000001:3.0:1788879339.002359:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879339.002361:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002363:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879339.002367:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879339/real 1788879339] req@ffff94e5045e7480 x1875765290153472/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879339.002375:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879339.002376:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879339.002378:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879339.002380:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879339.002382:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002383:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879339.002385:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879339.002389:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290153472/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879339.002396:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002398:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879339.002400:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002402:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879339.002404:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002405:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879339.002417:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879339.002419:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879339.002422:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879339.002424:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879339.002428:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290153472:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879339.002432:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879339.002433:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879339.002435:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879339.002437:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879339.002439:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879339.002441:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879339.002444:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290153600:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879339.002447:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879339.002448:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879339.002450:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879339.002451:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879339.002454:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002458:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879339.002460:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074559872 : -117763634991744 : ffff94e5045e6d80) 00000100:00000001:3.0:1788879339.002463:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879339.002472:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879339.002476:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879339.002478:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879339.002479:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002481:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879339.002484:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002487:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304950282 00000100:00000001:3.0:1788879339.002489:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304950826 00000100:00000001:3.0:1788879339.002492:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304950282 00000100:00000001:3.0:1788879339.002494:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304950826 00000100:00000001:3.0:1788879339.002496:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879339.002498:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002500:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879339.002504:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879339/real 1788879339] req@ffff94e5045e6d80 x1875765290153600/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879339.002511:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879339.002512:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879339.002514:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879339.002515:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879339.002517:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002519:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879339.002520:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879339.002524:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6d80 x1875765290153600/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879339.002531:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002533:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879339.002535:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002537:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879339.002538:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002539:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879339.002573:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879339.002577:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879339.002581:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879339.002584:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879339.002587:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290153600:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879339.002591:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879339.002593:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879339.002600:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879339.002602:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879339.002604:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879339.002606:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879339.002609:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290153728:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879339.002612:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879339.002614:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879339.002615:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879339.002617:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879339.002620:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002626:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879339.002627:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:3.0:1788879339.002632:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879339.002647:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879339.002654:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879339.002657:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879339.002659:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002663:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879339.002667:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002672:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304950282 00000100:00000001:3.0:1788879339.002676:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304950826 00000100:00000001:3.0:1788879339.002679:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304950282 00000100:00000001:3.0:1788879339.002682:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304950826 00000100:00000001:3.0:1788879339.002686:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879339.002689:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002693:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879339.002700:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879339/real 1788879339] req@ffff94e5045e4e00 x1875765290153728/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879339.002713:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879339.002716:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879339.002720:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879339.002722:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879339.002724:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002727:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879339.002729:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879339.002736:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290153728/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879339.002744:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002747:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879339.002749:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002751:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879339.002752:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002754:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879339.002784:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879339.002787:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879339.002792:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879339.002795:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879339.002799:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290153728:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879339.002803:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002806:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879339.002808:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879339.002810:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879339.002813:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879339.002815:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879339.002817:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879339.002820:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879339.002822:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879339.002825:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879339.002827:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002829:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879339.002831:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879339.002832:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879339.002834:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879339.002836:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879339.002837:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879339.002839:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879339.002840:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879339.002842:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879339.002844:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002845:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879339.002847:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879339.002848:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879339.002850:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879339.002851:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879339.002853:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879339.002854:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879339.002855:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879339.002858:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879339.002859:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002861:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879339.002865:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879339.002867:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879339.002870:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879339.002872:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879339.065008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879339.065018:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879339.065021:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879339.065029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879339.065034:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879339.065039:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879339.065043:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879339.065047:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879339.065052:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879339.769042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879339.769050:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879339.769057:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879339.769059:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879339.769100:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879339.769103:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879339.769107:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879340.089093:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879340.089103:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879340.089106:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879340.089112:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879340.089117:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879340.089122:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879340.089125:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879340.089128:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879340.089132:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879340.792977:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879340.792982:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879340.792986:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879340.792987:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879340.793039:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879340.793043:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879340.793046:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879341.112973:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879341.112980:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879341.112982:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879341.112988:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879341.112992:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879341.112996:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879341.113011:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879341.113013:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879341.113016:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879341.817095:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879341.817104:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879341.817114:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879341.817117:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879341.817177:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879341.817182:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879341.817188:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879342.136977:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879342.136984:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879342.136986:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879342.136992:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879342.136995:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879342.136998:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879342.137001:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879342.137004:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879342.137007:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879342.841029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879342.841037:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879342.841044:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879342.841045:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879342.841087:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879342.841091:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879342.841095:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879343.161025:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879343.161033:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879343.161036:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879343.161043:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879343.161047:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879343.161052:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879343.161055:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879343.161058:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879343.161061:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879343.864988:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879343.864993:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879343.864997:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879343.865000:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879343.865003:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879343.865006:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879343.865008:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879343.865010:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879343.865012:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879343.865014:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879343.865032:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879343.865034:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879343.865036:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879344.121003:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879344.121012:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879344.121014:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879344.121017:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879344.121021:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879344.121027:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10282 00000100:00080000:3.0:1788879344.121035:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879344.121038:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879344.121039:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879344.121046:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879344.121048:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879344.121051:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879344.121053:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879344.121055:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879344.121056:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879344.121060:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879344.121061:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879344.121064:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879344.121067:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879344.121069:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879344.121071:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879344.121073:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879344.121084:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879344.121085:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879344.121088:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879344.121090:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879344.121094:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879344.121104:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879344.121113:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290153856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879344.121121:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879344.121125:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879344.121168:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879344.121172:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879344.121175:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879344.121176:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879344.121178:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879344.121181:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879344.121183:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10282 00000100:00080000:3.0:1788879344.121186:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879344.121188:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879344.121189:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879344.121192:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879344.121194:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879344.121197:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879344.121199:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879344.121201:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879344.121203:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879344.121207:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879344.121209:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879344.121211:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879344.121217:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879344.121219:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879344.121221:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879344.121224:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:3.0:1788879344.121225:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879344.121231:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00000001:3.0:1788879344.121232:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121238:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879344.121238:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121244:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788879344.121247:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879344.121248:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879344.121250:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879344.121254:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879344.121256:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879344.121258:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879344.121259:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879344.121265:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879344.121267:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879344.121277:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879344.121279:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290153984/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879344.121291:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290153856:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879344.121293:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879344.121299:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879344.121306:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121308:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788879344.121310:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879344.121315:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1788879344.121316:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879344.121321:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1788879344.121323:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879344.121324:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1788879344.121329:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879344.121330:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879344.121335:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10282 02000000:00000001:1.0:1788879344.121337:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879344.121339:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879344.121343:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879344.121344:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879344.121347:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879344.121348:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879344.121350:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879344.121352:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879344.121354:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879344.121356:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879344.121359:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879344.121361:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879344.121362:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879344.121365:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879344.121367:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879344.121369:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879344.121370:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000008:00100000:3.0:1788879344.121372:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879344.121374:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000008:00000001:3.0:1788879344.121377:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121381:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879344.121381:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879344.121391:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879344.121393:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879344.121397:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879344.121400:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879344.121403:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879344.121412:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879344.121420:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290154112/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788879344.121427:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879344.121440:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879344.121442:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879344.121446:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121449:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879344.121453:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121454:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121461:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:3.0:1788879344.121464:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879344.121468:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879344.121470:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879344.121474:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304955945 00000100:00000001:3.0:1788879344.121475:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879344.121479:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879344.121484:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879344.121488:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788879344.121489:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10282 00000100:00080000:3.0:1788879344.121499:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879344.121500:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304955945 00000100:00000001:3.0:1788879344.121503:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879344.121508:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879344.121509:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879344.121515:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788879344.121534:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879344.121539:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879344.121540:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879344.121543:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879344.121562:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879344.121571:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879344/real 1788879344] req@ffff94e50449f480 x1875765290153856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788879344.121574:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879344.121581:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879344.121584:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:1.0:1788879344.121585:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879344.121589:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879344.121594:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:3.0:1788879344.121595:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879344.121598:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879344.121599:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121601:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879344.121603:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879344.121606:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788879344.121608:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879344.121609:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879344.121614:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879344.121620:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290153856/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879344.121620:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879344.121624:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879344.121630:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879344.121652:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121661:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879344.121662:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879344.121665:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788879344.121666:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121674:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879344.121675:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879344.121679:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879344.121680:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788879344.121684:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121685:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.121692:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879344.121694:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879344.121694:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879344.121699:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788879344.121706:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290154240/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879344.121709:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879344.121716:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879344.121726:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290153856:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879344.121728:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879344.121735:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121736:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879344.121739:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121741:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879344.121745:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879344.121749:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879344.121753:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879344.121755:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879344.121757:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879344.121762:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879344.121765:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879344.121768:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879344.121770:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.121773:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.121780:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879344.121783:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121785:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879344.121786:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879344.121788:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879344.121790:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879344.121791:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879344.121793:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879344.121795:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879344.121798:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879344.121801:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290153984:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879344.121804:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879344.121806:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879344.121807:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879344.121809:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879344.121812:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121818:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879344.121820:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:1.0:1788879344.121825:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879344.121838:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879344.121842:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879344.121844:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879344.121845:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121847:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879344.121849:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121853:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304955403 00000100:00000001:1.0:1788879344.121856:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304955945 00000100:00000001:1.0:1788879344.121858:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304955403 00000100:00000001:1.0:1788879344.121861:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304955945 00000100:00000001:1.0:1788879344.121863:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879344.121882:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121883:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879344.121887:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879344/real 1788879344] req@ffff94e50449c700 x1875765290153984/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879344.121894:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879344.121895:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879344.121897:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879344.121898:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879344.121899:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.121900:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879344.121902:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879344.121905:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290153984/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879344.121910:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.121911:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879344.121913:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.121914:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879344.121915:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.121916:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879344.121926:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879344.121928:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879344.121930:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879344.121932:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879344.121934:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290153984:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879344.121937:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879344.121937:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879344.121938:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879344.121940:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879344.121941:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879344.121943:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879344.121945:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290154112:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879344.121947:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879344.121976:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879344.121977:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879344.121979:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879344.121982:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.121989:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879344.121990:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:1.0:1788879344.121995:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879344.122009:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879344.122014:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879344.122017:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879344.122019:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.122021:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879344.122023:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.122027:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304955402 00000100:00000001:1.0:1788879344.122031:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304955945 00000100:00000001:1.0:1788879344.122034:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304955402 00000100:00000001:1.0:1788879344.122036:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304955945 00000100:00000001:1.0:1788879344.122039:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879344.122041:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.122044:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879344.122048:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879344/real 1788879344] req@ffff94e50449e300 x1875765290154112/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879344.122055:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879344.122056:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879344.122058:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879344.122060:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879344.122061:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.122062:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879344.122063:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879344.122066:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290154112/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879344.122076:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.122078:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879344.122080:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.122081:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879344.122082:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.122083:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879344.122092:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879344.122102:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879344.122104:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879344.122106:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879344.122108:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290154112:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879344.122111:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879344.122112:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879344.122113:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879344.122115:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879344.122117:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879344.122118:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879344.122120:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290154240:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879344.122123:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879344.122140:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879344.122142:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879344.122145:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879344.122149:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.122157:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879344.122159:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:1.0:1788879344.122164:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879344.122181:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879344.122188:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879344.122192:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879344.122194:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.122197:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879344.122200:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.122205:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304955402 00000100:00000001:1.0:1788879344.122209:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304955945 00000100:00000001:1.0:1788879344.122213:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304955402 00000100:00000001:1.0:1788879344.122216:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304955945 00000100:00000001:1.0:1788879344.122219:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879344.122221:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.122223:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879344.122228:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879344/real 1788879344] req@ffff94e50449c000 x1875765290154240/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879344.122235:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879344.122237:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879344.122239:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879344.122241:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879344.122242:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.122244:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879344.122245:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879344.122248:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290154240/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879344.122258:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.122261:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879344.122265:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.122267:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879344.122268:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.122271:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879344.122287:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879344.122291:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879344.122296:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879344.122300:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879344.122306:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290154240:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879344.122312:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.122315:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879344.122317:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879344.122318:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879344.122321:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879344.122323:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879344.122324:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879344.122326:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879344.122328:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879344.122330:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879344.122331:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.122333:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879344.122334:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879344.122335:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879344.122337:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879344.122338:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879344.122339:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879344.122340:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879344.122341:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879344.122342:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879344.122344:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.122345:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879344.122346:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879344.122347:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879344.122348:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879344.122349:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879344.122350:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879344.122352:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879344.122353:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879344.122354:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879344.122355:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.122357:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.122360:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879344.122362:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.122363:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879344.122365:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879344.184979:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879344.184988:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879344.184991:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879344.184999:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879344.185004:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879344.185008:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.185012:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879344.185016:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879344.185020:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879344.888986:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879344.888991:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879344.888995:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879344.888996:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879344.889038:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879344.889041:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879344.889044:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879345.208970:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879345.208977:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879345.208980:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879345.208986:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879345.208990:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879345.208993:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879345.208997:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879345.209000:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879345.209003:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879345.913035:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879345.913043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879345.913049:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879345.913051:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879345.913121:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879345.913125:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879345.913129:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879346.232979:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879346.232987:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879346.232990:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879346.232996:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879346.233001:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879346.233005:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879346.233008:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879346.233011:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879346.233014:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879346.937024:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879346.937032:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879346.937035:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879346.937039:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879346.937044:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879346.937047:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879346.937049:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879346.937052:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879346.937055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879346.937057:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879346.937081:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879346.937084:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879346.937087:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879347.256998:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879347.257005:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879347.257007:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879347.257013:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879347.257017:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879347.257021:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879347.257023:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879347.257026:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879347.257028:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879347.960985:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879347.960990:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879347.960994:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879347.960995:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879347.961035:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879347.961037:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879347.961040:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879348.280990:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879348.280995:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879348.280997:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879348.281002:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879348.281005:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879348.281008:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879348.281010:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879348.281012:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879348.281014:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879348.985046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879348.985055:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879348.985064:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879348.985067:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879348.985133:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879348.985138:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879348.985142:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879349.240990:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879349.241000:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879349.241002:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879349.241005:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879349.241010:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879349.241016:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10288 00000100:00080000:1.0:1788879349.241025:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879349.241027:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879349.241029:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879349.241036:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879349.241038:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879349.241041:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879349.241043:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879349.241045:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879349.241047:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879349.241050:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879349.241052:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879349.241054:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879349.241057:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879349.241059:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879349.241061:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879349.241062:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879349.241072:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879349.241075:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879349.241077:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879349.241080:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879349.241084:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879349.241094:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879349.241121:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009880 x1875765290154368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879349.241130:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879349.241152:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879349.241175:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879349.241179:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879349.241184:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879349.241186:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879349.241188:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879349.241192:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879349.241196:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10288 00000100:00080000:1.0:1788879349.241202:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879349.241206:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879349.241208:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879349.241213:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879349.241215:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879349.241218:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879349.241220:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1788879349.241221:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879349.241229:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879349.241230:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879349.241232:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879349.241238:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1788879349.241241:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879349.241244:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879349.241248:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241250:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879349.241253:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879349.241262:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879349.241265:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879349.241268:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879349.241274:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879349.241277:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879349.241280:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879349.241289:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879349.241290:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290154368:192.168.203.129@tcp:250:kworker.0 00000008:00000001:1.0:1788879349.241298:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241306:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788879349.241307:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879349.241316:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879349.241319:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879349.241322:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879349.241324:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879349.241328:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788879349.241330:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879349.241335:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879349.241338:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879349.241339:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879349.241355:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879349.241379:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008380 x1875765290154496/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879349.241387:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879349.241389:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879349.241393:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241393:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879349.241397:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286514304 : -117763423037312 : ffff94e511009880) 00000100:00080000:1.0:1788879349.241399:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879349.241405:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1788879349.241406:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879349.241410:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879349.241412:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879349.241416:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879349.241418:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10288 00000100:00080000:1.0:1788879349.241423:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879349.241425:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879349.241426:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879349.241429:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879349.241431:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879349.241433:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879349.241436:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879349.241438:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879349.241439:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879349.241453:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879349.241456:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879349.241458:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879349.241461:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879349.241463:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879349.241465:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879349.241467:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879349.241469:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1788879349.241470:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879349.241475:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241476:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304961065 02000000:00000001:1.0:1788879349.241513:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879349.241516:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879349.241518:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879349.241520:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879349.241520:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1788879349.241525:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879349.241533:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241534:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304961065 00000100:00100000:1.0:1788879349.241538:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009180 x1875765290154624/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879349.241543:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879349.241548:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879349.241550:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879349.241553:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241555:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879349.241558:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241560:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879349.241562:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879349.241564:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788879349.241568:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879349.241570:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879349.241572:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879349.241573:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879349.241574:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879349.241577:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00080000:1.0:1788879349.241579:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10288 00000100:00000001:3.0:1788879349.241581:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1788879349.241585:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879349.241586:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879349.241590:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879349.241591:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879349.241593:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879349.241594:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879349.241596:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879349.241598:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879349.241601:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:3.0:1788879349.241601:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879349.241604:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1788879349.241606:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:3.0:1788879349.241609:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879349.241610:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879349.241614:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1788879349.241617:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290154496:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879349.241618:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879349.241625:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1788879349.241627:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:3.0:1788879349.241628:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1788879349.241631:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879349.241632:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879349.241634:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879349.241636:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879349.241639:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879349.241641:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879349.241643:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879349.241646:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879349.241647:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879349.241649:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241650:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879349.241651:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241655:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508928 : -117763423042688 : ffff94e511008380) 02000000:00000001:1.0:1788879349.241658:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879349.241661:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1788879349.241662:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879349.241665:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879349.241667:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879349.241669:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879349.241674:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879349.241678:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009f80 x1875765290154752/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879349.241679:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241686:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304960523 00000100:00000001:1.0:1788879349.241689:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879349.241690:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304961065 00000100:00000001:1.0:1788879349.241694:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241696:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304960523 00000100:00000001:3.0:1788879349.241700:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304961065 00000100:00000001:1.0:1788879349.241701:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241703:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879349.241707:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241709:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879349.241711:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879349.241712:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879349.241714:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879349.241726:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879349.241728:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879349.241731:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290154624:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879349.241734:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879349.241735:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879349.241736:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879349.241738:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879349.241740:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241743:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879349.241744:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286512512 : -117763423039104 : ffff94e511009180) 00000100:00000001:3.0:1788879349.241747:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879349.241753:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241755:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304960521 00000100:00000001:3.0:1788879349.241757:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304961065 00000100:00000001:3.0:1788879349.241759:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304960521 00000100:00000001:3.0:1788879349.241761:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304961065 00000100:00000001:3.0:1788879349.241762:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879349.241763:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241765:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241766:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879349.241768:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879349.241769:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788879349.241771:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879349.241771:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879349.241772:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879349.241773:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879349.241774:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879349.241776:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879349.241777:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879349.241778:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879349.241781:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290154752:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879349.241795:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879349.241796:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879349.241797:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879349.241799:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879349.241800:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241803:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879349.241804:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286516096 : -117763423035520 : ffff94e511009f80) 00000100:00000001:3.0:1788879349.241807:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879349.241812:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241814:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304960521 00000100:00000001:3.0:1788879349.241815:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304961065 00000100:00000001:3.0:1788879349.241817:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304960521 00000100:00000001:3.0:1788879349.241818:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304961065 00000100:00000001:3.0:1788879349.241820:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879349.241821:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241823:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241824:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879349.241825:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879349.241826:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788879349.241828:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879349.241828:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879349.241829:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879349.241831:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879349.305006:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879349.305012:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879349.305014:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879349.305019:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879349.305023:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879349.305026:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879349.305029:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879349.305031:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879349.305034:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879350.009039:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879350.009047:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879350.009054:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879350.009056:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879350.009083:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879350.009086:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879350.009089:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879350.329052:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879350.329062:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879350.329064:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879350.329071:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879350.329076:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879350.329092:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879350.329096:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879350.329099:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879350.329102:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879351.033051:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879351.033059:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879351.033066:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879351.033068:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879351.033123:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879351.033127:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879351.033130:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879351.353037:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879351.353044:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879351.353046:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879351.353051:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879351.353054:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879351.353058:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879351.353060:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879351.353063:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879351.353065:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879352.057014:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879352.057022:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879352.057030:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879352.057032:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879352.057070:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879352.057073:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879352.057075:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788879352.313225:0:2350:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1788879352.313279:0:2350:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788879352.313302:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879352.313319:0:2350:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879352.313345:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879352.313349:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879352.313352:0:2350:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879352.313363:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879352.313369:0:2350:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879352.313373:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879352.313374:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879352.313376:0:2350:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879352.313383:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879352.313389:0:2350:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879352.313393:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879352.313394:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879352.313396:0:2350:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879352.313403:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788879352.313404:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879352.313409:0:2350:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879352.313412:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879352.313413:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879352.313416:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879352.313417:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879352.313420:0:2350:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:3.0:1788879352.313432:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879349/real 1788879352] req@ffff94e511009f80 x1875765290154752/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879352.313451:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879352.313453:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879352.313475:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879352.313478:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879352.313481:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313484:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879352.313487:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879352.313492:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009f80 x1875765290154752/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879352.313503:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313508:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879352.313512:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313514:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879352.313515:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313519:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879352.313541:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879352.313545:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879352.313551:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879352.313554:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879352.313560:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290154752:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879352.313566:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879352.313571:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879349/real 1788879352] req@ffff94e511008380 x1875765290154496/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879352.313581:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879352.313582:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879352.313586:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879352.313588:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879352.313590:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313592:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879352.313594:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879352.313597:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008380 x1875765290154496/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879352.313604:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313606:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879352.313609:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313611:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879352.313612:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313615:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879352.313638:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879352.313641:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879352.313647:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879352.313650:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879352.313653:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290154496:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879352.313658:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879352.313664:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879349/real 1788879352] req@ffff94e511009180 x1875765290154624/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879352.313673:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879352.313675:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879352.313678:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879352.313680:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879352.313681:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313683:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879352.313685:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879352.313689:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009180 x1875765290154624/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879352.313695:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313698:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879352.313701:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313703:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879352.313704:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313706:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879352.313731:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879352.313734:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879352.313738:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879352.313740:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879352.313743:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290154624:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879352.313748:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879352.313753:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879349/real 1788879352] req@ffff94e511009880 x1875765290154368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879352.313762:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879352.313764:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879352.313768:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879352.313771:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879352.313772:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313775:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879352.313776:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879352.313780:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009880 x1875765290154368/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879352.313787:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313789:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879352.313791:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313792:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879352.313794:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313795:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879352.313797:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879352.313799:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879352.313803:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879352.313806:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879352.313809:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290154368:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879352.313815:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313818:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879352.313821:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879352.313826:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879352.313835:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879352.313841:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879352.313844:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879352.313852:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879352.313856:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879352.313862:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879352.313864:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313866:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879352.313867:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879352.313868:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879352.313871:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879352.313872:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879352.313873:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879352.313876:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879352.313877:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879352.313880:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879352.313881:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313882:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879352.313884:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879352.313885:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879352.313887:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879352.313889:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879352.313890:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879352.313892:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879352.313893:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879352.313895:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879352.313896:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313898:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879352.313899:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879352.313931:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879352.313935:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879352.313937:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879352.313938:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879352.313941:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879352.313943:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879352.313945:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879352.313947:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.313951:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879352.314042:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879352.314046:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879352.314050:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879352.314053:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879352.376987:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879352.376993:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879352.376994:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879352.376998:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879352.377001:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879352.377004:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879352.377006:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879352.377008:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879352.377010:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879353.081034:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879353.081041:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879353.081046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879353.081049:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879353.081052:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879353.081055:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879353.081058:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879353.081060:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879353.081062:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879353.081063:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879353.081124:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879353.081128:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879353.081131:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879353.401091:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879353.401100:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879353.401102:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879353.401107:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879353.401110:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879353.401114:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879353.401117:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879353.401119:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879353.401122:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879354.105062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879354.105074:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879354.105082:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879354.105084:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879354.105118:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879354.105121:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879354.105125:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879354.361105:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879354.361117:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879354.361119:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879354.361124:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879354.361128:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879354.361136:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10293 00000100:00080000:3.0:1788879354.361148:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879354.361151:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879354.361152:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879354.361160:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879354.361163:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879354.361166:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879354.361168:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879354.361170:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879354.361171:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879354.361175:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879354.361177:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879354.361179:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879354.361182:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879354.361185:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879354.361187:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879354.361189:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879354.361199:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879354.361200:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879354.361204:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879354.361207:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879354.361212:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879354.361225:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879354.361237:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290154880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879354.361250:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879354.361255:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879354.361280:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879354.361284:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879354.361288:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879354.361290:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879354.361294:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879354.361299:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879354.361303:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10293 00000100:00080000:3.0:1788879354.361308:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879354.361316:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879354.361318:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879354.361322:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788879354.361324:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879354.361329:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879354.361330:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788879354.361333:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879354.361334:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879354.361339:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1788879354.361340:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879354.361345:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879354.361346:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879354.361355:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879354.361357:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1788879354.361363:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879354.361367:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879354.361368:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879354.361369:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879354.361378:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879354.361379:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879354.361388:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879354.361392:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:1.0:1788879354.361400:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290154880:192.168.203.129@tcp:250:kworker.0 00000008:00100000:3.0:1788879354.361402:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879354.361413:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.361418:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879354.361419:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879354.361426:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879354.361429:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879354.361433:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879354.361434:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879354.361459:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879354.361460:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879354.361466:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879354.361468:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879354.361474:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879354.361478:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879354.361491:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290155008/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879354.361502:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879354.361506:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879354.361506:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.361512:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:3.0:1788879354.361517:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.361522:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879354.361524:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879354.361531:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879354.361533:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879354.361536:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879354.361539:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879354.361545:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10293 00000100:00080000:3.0:1788879354.361551:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879354.361553:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879354.361555:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879354.361559:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879354.361561:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879354.361564:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879354.361567:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879354.361569:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879354.361571:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879354.361575:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879354.361579:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879354.361586:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879354.361586:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879354.361595:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879354.361602:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879354.361605:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879354.361609:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00100000:3.0:1788879354.361611:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879354.361619:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.361623:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879354.361624:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.361627:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.361634:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879354.361637:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879354.361642:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.361644:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879354.361645:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879354.361650:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879354.361654:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.361657:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304966185 00000100:00000001:3.0:1788879354.361665:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.361673:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00100000:3.0:1788879354.361678:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290155136/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879354.361686:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304966185 00000100:00000001:1.0:1788879354.361694:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879354.361696:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879354.361703:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.361706:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879354.361712:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879354.361716:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.361718:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788879354.361726:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788879354.361736:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879354/real 1788879354] req@ffff94e50449ca80 x1875765290154880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879354.361737:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879354.361743:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879354.361746:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879354.361755:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879354.361764:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10293 00000400:00000001:1.0:1788879354.361775:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1788879354.361776:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000400:00000001:1.0:1788879354.361779:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879354.361784:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879354.361787:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879354.361790:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879354.361791:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879354.361793:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.361799:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1788879354.361803:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879354.361804:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788879354.361810:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879354.361815:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:1.0:1788879354.361818:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290154880/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788879354.361822:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879354.361831:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879354.361836:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879354.361851:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879354.361852:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879354.361861:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.361863:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788879354.361870:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879354.361873:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879354.361877:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879354.361879:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879354.361884:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879354.361886:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879354.361895:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879354.361896:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000008:00100000:3.0:1788879354.361899:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879354.361941:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879354.361949:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879354.361957:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879354.361961:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879354.361968:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290154880:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879354.361974:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1788879354.361976:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.361983:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879354.361984:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.361989:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879354.361998:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879354.361998:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879354.362003:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879354.362008:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1788879354.362010:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:1.0:1788879354.362015:0:2361:0:(genops.c:1216:class_import_put()) Process entered 02000000:00000001:3.0:1788879354.362016:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879354.362020:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879354.362022:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879354.362032:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879354.362034:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879354.362038:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879354.362040:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362043:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362051:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879354.362054:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.362057:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879354.362059:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879354.362061:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879354.362063:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879354.362065:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879354.362068:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879354.362071:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879354.362074:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879354.362078:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290155008:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879354.362084:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879354.362087:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879354.362089:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879354.362091:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879354.362095:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.362105:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879354.362107:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:1.0:1788879354.362113:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879354.362122:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879354.362134:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290155264/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788879354.362142:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879354.362151:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879354.362153:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879354.362158:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879354.362159:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.362165:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879354.362170:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.362173:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879354.362182:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.362189:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304965644 00000100:00000001:1.0:1788879354.362193:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304966185 00000100:00000001:1.0:1788879354.362197:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304965644 00000100:00000001:1.0:1788879354.362199:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304966185 00000100:00000001:1.0:1788879354.362202:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879354.362206:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.362208:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879354.362217:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879354/real 1788879354] req@ffff94e50449f100 x1875765290155008/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879354.362229:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879354.362230:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879354.362233:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879354.362236:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879354.362238:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362240:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879354.362241:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879354.362245:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290155008/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879354.362253:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362255:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879354.362258:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362261:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879354.362262:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362264:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879354.362280:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879354.362283:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879354.362287:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879354.362290:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879354.362293:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290155008:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879354.362297:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879354.362299:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879354.362301:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879354.362303:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879354.362306:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879354.362308:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879354.362311:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290155136:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879354.362315:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879354.362318:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879354.362320:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879354.362322:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879354.362327:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.362334:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879354.362336:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:1.0:1788879354.362340:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879354.362355:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879354.362360:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879354.362363:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879354.362364:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.362367:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879354.362371:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.362374:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304965642 00000100:00000001:1.0:1788879354.362377:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304966185 00000100:00000001:1.0:1788879354.362379:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304965642 00000100:00000001:1.0:1788879354.362381:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304966185 00000100:00000001:1.0:1788879354.362401:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879354.362404:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.362406:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879354.362412:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879354/real 1788879354] req@ffff94e50449c000 x1875765290155136/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879354.362425:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879354.362427:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879354.362430:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879354.362433:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879354.362435:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362438:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879354.362440:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879354.362480:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290155136/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879354.362489:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362492:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879354.362495:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362497:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879354.362499:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362501:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879354.362517:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879354.362520:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879354.362524:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879354.362527:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879354.362532:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290155136:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879354.362536:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362538:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879354.362540:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879354.362542:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879354.362546:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879354.362548:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879354.362549:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879354.362552:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879354.362554:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879354.362557:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879354.362558:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362560:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879354.362562:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879354.362563:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879354.362565:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879354.362567:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879354.362568:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879354.362571:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879354.362572:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879354.362573:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879354.362575:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362577:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362580:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879354.362582:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.362584:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879354.362586:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879354.362588:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879354.362590:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879354.362592:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879354.362594:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879354.362596:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879354.362598:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879354.362602:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290155264:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879354.362606:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879354.362608:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879354.362610:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879354.362612:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879354.362616:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.362623:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879354.362624:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:1.0:1788879354.362628:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879354.362642:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879354.362648:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879354.362650:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879354.362652:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.362654:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879354.362657:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.362661:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304965642 00000100:00000001:1.0:1788879354.362664:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304966185 00000100:00000001:1.0:1788879354.362667:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304965642 00000100:00000001:1.0:1788879354.362669:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304966185 00000100:00000001:1.0:1788879354.362672:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879354.362675:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.362677:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879354.362682:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879354/real 1788879354] req@ffff94e50449e300 x1875765290155264/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879354.362692:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879354.362693:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879354.362696:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879354.362698:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879354.362700:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362702:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879354.362704:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879354.362709:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290155264/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879354.362718:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362721:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879354.362724:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362726:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879354.362728:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362730:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879354.362743:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879354.362747:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879354.362752:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879354.362754:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879354.362757:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290155264:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879354.362760:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362762:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879354.362763:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879354.362764:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879354.362766:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879354.362767:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879354.362768:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879354.362770:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879354.362771:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879354.362773:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879354.362774:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362776:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.362779:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879354.362780:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.362781:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879354.362783:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879354.425008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879354.425014:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879354.425017:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879354.425023:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879354.425027:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879354.425031:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.425034:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879354.425037:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879354.425040:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879355.129064:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879355.129074:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879355.129081:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879355.129083:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879355.129116:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879355.129120:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879355.129124:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879355.448987:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879355.448994:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879355.449002:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879355.449007:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879355.449010:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879355.449013:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879355.449015:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879355.449017:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879355.449019:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879356.153021:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879356.153027:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879356.153030:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879356.153032:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879356.153036:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879356.153038:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879356.153039:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879356.153041:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879356.153043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879356.153044:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879356.153097:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879356.153100:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879356.153103:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879356.473057:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879356.473066:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879356.473069:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879356.473076:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879356.473081:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879356.473085:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879356.473106:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879356.473109:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879356.473112:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879357.176987:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879357.176992:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879357.176996:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879357.176997:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879357.177071:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879357.177074:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879357.177077:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879357.497003:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879357.497012:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879357.497014:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879357.497020:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879357.497024:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879357.497028:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879357.497032:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879357.497035:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879357.497039:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879358.201000:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879358.201006:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879358.201010:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879358.201011:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879358.201039:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879358.201041:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879358.201043:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879358.520964:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879358.520972:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879358.520974:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879358.520979:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879358.520983:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879358.520986:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879358.520989:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879358.520992:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879358.520994:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879359.224978:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879359.224983:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879359.224987:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879359.224988:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879359.225004:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879359.225007:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879359.225010:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879359.480991:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879359.480999:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879359.481002:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879359.481005:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879359.481008:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879359.481014:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10298 00000100:00080000:3.0:1788879359.481023:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879359.481025:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879359.481027:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879359.481033:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879359.481035:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879359.481038:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879359.481040:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879359.481041:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879359.481042:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879359.481046:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879359.481047:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879359.481049:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879359.481052:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879359.481054:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481056:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879359.481058:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879359.481068:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879359.481069:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481072:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879359.481074:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879359.481078:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481087:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879359.481094:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290155392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879359.481102:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879359.481105:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481118:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879359.481121:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879359.481123:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879359.481125:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879359.481126:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879359.481128:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879359.481129:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10298 00000100:00080000:3.0:1788879359.481131:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879359.481133:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879359.481134:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879359.481135:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879359.481136:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879359.481138:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879359.481139:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879359.481140:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879359.481141:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879359.481143:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879359.481144:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879359.481145:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879359.481149:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879359.481150:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481151:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879359.481155:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879359.481157:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481159:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879359.481161:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879359.481162:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481164:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879359.481165:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879359.481166:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481169:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879359.481173:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290155520/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879359.481178:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879359.481180:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481182:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481184:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1788879359.481184:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879359.481189:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879359.481190:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879359.481191:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879359.481193:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1788879359.481194:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879359.481204:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879359.481206:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1788879359.481207:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10298 00000100:00000001:1.0:1788879359.481209:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879359.481211:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879359.481215:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879359.481216:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879359.481217:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879359.481221:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788879359.481222:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879359.481225:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1788879359.481226:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879359.481227:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879359.481231:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00100000:1.0:1788879359.481233:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290155392:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879359.481233:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879359.481236:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879359.481242:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788879359.481243:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788879359.481246:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788879359.481247:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879359.481249:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879359.481250:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879359.481254:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879359.481255:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879359.481258:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481269:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879359.481270:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879359.481271:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788879359.481273:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481275:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879359.481279:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879359.481280:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481281:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879359.481282:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879359.481283:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481287:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481288:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00100000:3.0:1788879359.481291:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290155648/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879359.481292:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:1.0:1788879359.481299:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879359.481303:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879359.481305:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481308:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879359.481309:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879359.481312:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879359.481313:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879359.481323:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879359.481326:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879359.481327:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10298 00000100:00080000:3.0:1788879359.481330:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879359.481332:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879359.481334:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:1.0:1788879359.481335:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:3.0:1788879359.481348:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879359.481349:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879359.481351:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879359.481353:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879359.481354:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879359.481355:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879359.481356:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879359.481361:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1788879359.481362:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879359.481365:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481366:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481370:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879359.481370:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879359.481375:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879359.481377:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879359.481377:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481383:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879359.481384:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304971305 00000008:00100000:3.0:1788879359.481384:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879359.481390:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481391:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481395:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788879359.481395:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879359.481398:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481400:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879359.481401:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879359.481403:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879359.481407:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879359.481422:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290155776/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879359.481428:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304971305 00000100:00000001:3.0:1788879359.481430:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879359.481434:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481436:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879359.481436:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481452:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879359.481454:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481456:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879359.481463:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879359/real 1788879359] req@ffff94e50449c700 x1875765290155392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879359.481472:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879359.481474:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879359.481478:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879359.481480:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879359.481481:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481484:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879359.481491:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879359.481494:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290155392/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879359.481499:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481501:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879359.481503:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481504:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879359.481505:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481507:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879359.481508:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879359.481510:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879359.481513:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879359.481515:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879359.481518:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290155392:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879359.481521:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879359.481522:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879359.481523:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879359.481524:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879359.481525:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879359.481527:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879359.481529:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290155520:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879359.481531:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879359.481532:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879359.481533:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879359.481534:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879359.481537:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481541:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879359.481542:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:1.0:1788879359.481545:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879359.481553:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879359.481556:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879359.481557:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879359.481558:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481560:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879359.481561:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481564:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304970763 00000100:00000001:1.0:1788879359.481566:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304971305 00000100:00000001:1.0:1788879359.481568:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304970763 00000100:00000001:1.0:1788879359.481569:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304971305 00000100:00000001:1.0:1788879359.481571:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879359.481572:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481573:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879359.481575:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879359/real 1788879359] req@ffff94e50449f480 x1875765290155520/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879359.481580:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879359.481581:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879359.481582:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879359.481583:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879359.481584:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481585:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879359.481586:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879359.481588:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290155520/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879359.481593:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481594:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879359.481595:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481596:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879359.481597:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481598:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879359.481607:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879359.481609:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879359.481611:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879359.481612:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879359.481614:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290155520:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879359.481617:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481619:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879359.481620:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879359.481622:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879359.481624:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879359.481625:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879359.481627:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879359.481630:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879359.481631:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879359.481632:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879359.481633:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481634:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879359.481635:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879359.481636:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879359.481637:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879359.481638:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879359.481639:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879359.481640:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879359.481641:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879359.481642:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879359.481643:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481644:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481649:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879359.481650:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481652:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879359.481653:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879359.481653:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879359.481654:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879359.481655:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879359.481656:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879359.481657:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879359.481659:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879359.481661:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290155648:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879359.481663:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879359.481664:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879359.481665:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879359.481666:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879359.481668:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481671:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879359.481672:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:1.0:1788879359.481674:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879359.481686:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879359.481689:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879359.481690:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879359.481691:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481693:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879359.481694:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481697:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304970761 00000100:00000001:1.0:1788879359.481698:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304971305 00000100:00000001:1.0:1788879359.481700:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304970761 00000100:00000001:1.0:1788879359.481702:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304971305 00000100:00000001:1.0:1788879359.481703:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879359.481704:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481705:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879359.481708:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879359/real 1788879359] req@ffff94e50449f800 x1875765290155648/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879359.481712:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879359.481713:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879359.481714:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879359.481715:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879359.481716:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481717:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879359.481718:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879359.481721:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290155648/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879359.481725:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481726:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879359.481728:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481729:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879359.481730:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481731:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879359.481740:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879359.481741:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879359.481743:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879359.481745:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879359.481747:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290155648:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879359.481749:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879359.481750:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879359.481750:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879359.481752:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879359.481753:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879359.481754:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879359.481755:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290155776:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879359.481757:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879359.481758:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879359.481759:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879359.481760:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879359.481762:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481765:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879359.481766:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:1.0:1788879359.481768:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879359.481774:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879359.481776:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879359.481777:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879359.481778:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481779:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879359.481781:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481783:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304970761 00000100:00000001:1.0:1788879359.481784:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304971305 00000100:00000001:1.0:1788879359.481786:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304970761 00000100:00000001:1.0:1788879359.481787:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304971305 00000100:00000001:1.0:1788879359.481789:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879359.481790:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481791:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879359.481793:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879359/real 1788879359] req@ffff94e50449d880 x1875765290155776/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879359.481798:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879359.481798:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879359.481799:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879359.481800:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879359.481801:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481802:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879359.481803:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879359.481805:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290155776/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879359.481809:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481811:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879359.481812:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481813:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879359.481814:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481815:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879359.481824:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879359.481825:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879359.481827:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879359.481828:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879359.481830:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290155776:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879359.481833:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481834:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879359.481835:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879359.481836:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879359.481837:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879359.481838:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879359.481839:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879359.481841:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879359.481841:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879359.481843:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879359.481844:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481845:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879359.481845:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879359.481846:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879359.481847:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879359.481848:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879359.481849:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879359.481850:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879359.481850:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879359.481851:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879359.481852:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481853:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.481855:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879359.481855:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.481856:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879359.481858:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879359.544959:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879359.544965:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879359.544967:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879359.544971:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879359.544974:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879359.544977:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.544979:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879359.544981:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879359.544983:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879360.248981:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879360.248987:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879360.248991:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879360.248992:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879360.249008:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879360.249010:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879360.249012:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879360.568978:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879360.568983:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879360.568985:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879360.568989:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879360.568991:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879360.568995:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879360.568997:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879360.568999:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879360.569001:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879361.273044:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879361.273052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879361.273058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879361.273060:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879361.273189:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879361.273195:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879361.273200:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879361.593005:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879361.593013:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879361.593016:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879361.593038:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879361.593043:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879361.593047:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879361.593051:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879361.593055:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879361.593058:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879362.297103:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879362.297110:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879362.297116:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879362.297118:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879362.297182:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879362.297186:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879362.297190:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879362.617007:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879362.617026:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879362.617030:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879362.617038:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879362.617043:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879362.617048:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879362.617052:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879362.617056:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879362.617060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879363.321003:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879363.321019:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879363.321025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879363.321028:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879363.321030:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879363.321033:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879363.321036:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879363.321038:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879363.321040:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879363.321042:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879363.321062:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879363.321064:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879363.321067:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879363.640982:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879363.640989:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879363.640991:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879363.640997:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879363.641000:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879363.641003:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879363.641006:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879363.641009:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879363.641013:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879364.345058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879364.345067:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879364.345074:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879364.345076:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879364.345129:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879364.345133:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879364.345136:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879364.600987:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879364.600999:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879364.601001:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879364.601005:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879364.601010:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879364.601018:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10303 00000100:00080000:3.0:1788879364.601029:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879364.601032:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879364.601033:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879364.601053:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879364.601056:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879364.601061:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879364.601063:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879364.601065:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879364.601066:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879364.601070:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879364.601071:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879364.601074:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879364.601077:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879364.601080:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879364.601082:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879364.601085:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879364.601100:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879364.601101:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879364.601105:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879364.601108:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879364.601113:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879364.601130:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879364.601144:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290155904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879364.601159:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879364.601166:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879364.601185:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879364.601188:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879364.601192:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879364.601193:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879364.601194:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879364.601197:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879364.601199:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10303 00000100:00080000:3.0:1788879364.601203:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879364.601204:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879364.601206:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879364.601208:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879364.601210:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879364.601212:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879364.601214:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879364.601215:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879364.601216:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879364.601218:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879364.601220:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879364.601221:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879364.601227:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879364.601228:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879364.601229:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879364.601234:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879364.601236:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879364.601238:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879364.601242:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879364.601243:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879364.601245:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879364.601246:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879364.601248:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601252:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879364.601253:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601258:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1788879364.601260:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290156032/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879364.601262:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879364.601266:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879364.601271:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879364.601272:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879364.601276:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879364.601278:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879364.601280:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879364.601282:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879364.601283:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879364.601288:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879364.601289:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879364.601293:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879364.601296:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788879364.601297:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290155904:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879364.601300:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879364.601305:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788879364.601306:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10303 02000000:00000001:1.0:1788879364.601309:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1788879364.601310:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788879364.601313:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879364.601314:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879364.601318:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879364.601318:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879364.601323:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879364.601324:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1788879364.601326:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879364.601326:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879364.601332:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879364.601334:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879364.601334:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879364.601337:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879364.601339:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879364.601340:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879364.601342:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879364.601343:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879364.601344:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879364.601346:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879364.601347:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879364.601348:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601349:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879364.601352:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 02000000:00000001:3.0:1788879364.601353:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879364.601356:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601359:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879364.601359:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879364.601363:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879364.601364:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879364.601369:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879364.601372:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290156160/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879364.601393:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879364.601395:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879364.601397:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879364.601399:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879364.601402:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879364.601404:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879364.601405:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879364.601418:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879364.601420:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10303 00000100:00080000:3.0:1788879364.601424:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879364.601425:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879364.601426:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879364.601429:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879364.601430:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879364.601432:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000800:00000100:1.0:1788879364.601436:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:3.0:1788879364.601436:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879364.601441:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879364.601442:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879364.601445:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879364.601448:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879364.601450:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879364.601452:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879364.601453:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788879364.601454:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879364.601457:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879364.601459:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00100000:3.0:1788879364.601460:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879364.601464:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788879364.601464:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601468:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879364.601469:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601475:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879364.601476:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879364.601479:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601482:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304976425 00000100:00000001:3.0:1788879364.601482:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879364.601483:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879364.601487:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879364.601493:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601495:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00100000:3.0:1788879364.601497:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290156288/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879364.601515:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879364.601516:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304976425 00000100:00000001:3.0:1788879364.601517:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879364.601523:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601525:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879364.601532:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879364.601534:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601536:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879364.601542:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879364/real 1788879364] req@ffff94e50449fb80 x1875765290155904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879364.601552:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879364.601554:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879364.601557:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879364.601559:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879364.601561:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.601563:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879364.601565:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879364.601568:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290155904/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879364.601583:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.601588:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879364.601591:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.601593:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879364.601594:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.601597:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879364.601600:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879364.601603:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879364.601609:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879364.601612:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879364.601640:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290155904:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879364.601646:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.601648:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879364.601650:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879364.601652:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879364.601656:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879364.601658:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879364.601660:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879364.601664:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879364.601665:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879364.601668:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879364.601669:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.601671:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.601677:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879364.601679:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601681:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879364.601682:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879364.601683:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879364.601684:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879364.601685:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879364.601687:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879364.601700:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879364.601701:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879364.601704:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290156032:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879364.601707:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879364.601708:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879364.601709:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879364.601711:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879364.601713:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601718:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879364.601720:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:1.0:1788879364.601724:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879364.601734:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879364.601737:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879364.601739:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879364.601740:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601741:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879364.601743:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601746:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304975883 00000100:00000001:1.0:1788879364.601748:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304976425 00000100:00000001:1.0:1788879364.601750:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304975883 00000100:00000001:1.0:1788879364.601752:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304976425 00000100:00000001:1.0:1788879364.601753:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879364.601754:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601766:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879364.601768:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879364/real 1788879364] req@ffff94e50449c380 x1875765290156032/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879364.601773:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879364.601774:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879364.601776:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879364.601777:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879364.601778:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.601779:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879364.601781:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879364.601783:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290156032/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879364.601792:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.601793:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879364.601795:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.601796:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879364.601797:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.601798:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879364.601812:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879364.601813:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879364.601815:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879364.601816:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879364.601818:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290156032:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879364.601821:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879364.601822:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879364.601823:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879364.601824:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879364.601826:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879364.601827:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879364.601829:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290156160:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879364.601830:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879364.601831:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879364.601832:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879364.601833:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879364.601835:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601839:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879364.601840:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:1.0:1788879364.601842:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879364.601848:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879364.601851:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879364.601852:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879364.601853:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601855:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879364.601856:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601858:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304975881 00000100:00000001:1.0:1788879364.601860:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304976425 00000100:00000001:1.0:1788879364.601862:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304975881 00000100:00000001:1.0:1788879364.601864:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304976425 00000100:00000001:1.0:1788879364.601865:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879364.601867:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601868:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879364.601870:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879364/real 1788879364] req@ffff94e50449d180 x1875765290156160/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879364.601874:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879364.601875:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879364.601876:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879364.601877:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879364.601878:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.601879:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879364.601880:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879364.601886:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290156160/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879364.601890:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.601892:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879364.601894:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.601895:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879364.601895:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.601896:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879364.601931:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879364.601933:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879364.601935:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879364.601937:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879364.601939:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290156160:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879364.601942:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879364.601943:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879364.601944:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879364.601946:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879364.601947:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879364.601949:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879364.601951:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290156288:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879364.601952:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879364.601953:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879364.601954:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879364.601955:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879364.601958:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601961:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879364.601962:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:1.0:1788879364.601965:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879364.601972:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879364.601975:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879364.601977:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879364.601978:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601979:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879364.601981:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601983:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304975882 00000100:00000001:1.0:1788879364.601985:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304976425 00000100:00000001:1.0:1788879364.601986:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304975882 00000100:00000001:1.0:1788879364.601988:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304976425 00000100:00000001:1.0:1788879364.601990:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879364.601991:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.601992:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879364.601995:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879364/real 1788879364] req@ffff94e50449df80 x1875765290156288/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879364.602005:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879364.602006:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879364.602007:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879364.602008:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879364.602009:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.602010:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879364.602011:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879364.602014:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290156288/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879364.602019:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.602020:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879364.602021:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.602022:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879364.602023:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.602024:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879364.602035:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879364.602037:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879364.602039:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879364.602040:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879364.602041:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290156288:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879364.602052:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.602054:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879364.602055:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879364.602056:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879364.602058:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879364.602059:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879364.602060:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879364.602061:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879364.602062:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879364.602064:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879364.602065:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.602067:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879364.602068:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879364.602069:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879364.602070:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879364.602071:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879364.602072:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879364.602073:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879364.602074:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879364.602075:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879364.602076:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.602077:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879364.602077:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879364.602078:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879364.602079:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879364.602080:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879364.602080:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879364.602081:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879364.602082:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879364.602083:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879364.602083:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.602084:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.602086:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879364.602087:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.602088:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879364.602090:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879364.664972:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879364.664979:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879364.664981:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879364.664985:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879364.664988:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879364.664991:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.664994:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879364.664996:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879364.664998:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879365.369042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879365.369049:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879365.369054:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879365.369056:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879365.369189:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879365.369198:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879365.369204:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879365.689057:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879365.689077:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879365.689080:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879365.689086:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879365.689092:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879365.689097:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879365.689100:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879365.689103:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879365.689106:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879366.393017:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879366.393029:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879366.393033:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879366.393037:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879366.393044:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879366.393058:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879366.393061:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879366.393063:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879366.393068:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879366.393070:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879366.393096:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879366.393100:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879366.393103:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879366.713053:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879366.713064:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879366.713066:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879366.713072:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879366.713075:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879366.713079:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879366.713082:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879366.713085:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879366.713104:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879367.417052:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879367.417066:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879367.417077:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879367.417080:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879367.417143:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879367.417147:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879367.417157:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879367.737027:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879367.737038:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879367.737040:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879367.737046:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879367.737051:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879367.737055:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879367.737058:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879367.737061:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879367.737064:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879368.441036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879368.441058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879368.441074:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879368.441077:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879368.441125:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879368.441130:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879368.441136:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879368.761007:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879368.761017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879368.761020:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879368.761027:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879368.761031:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879368.761036:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879368.761039:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879368.761043:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879368.761046:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879369.464986:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879369.464992:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879369.464997:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879369.464999:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879369.465070:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879369.465078:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879369.465085:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879369.721114:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879369.721135:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879369.721142:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879369.721150:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879369.721160:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879369.721175:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10308 00000100:00080000:1.0:1788879369.721197:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879369.721204:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879369.721226:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879369.721240:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879369.721245:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879369.721250:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879369.721254:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879369.721258:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879369.721260:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879369.721267:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879369.721270:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879369.721275:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879369.721280:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879369.721285:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879369.721288:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879369.721292:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879369.721312:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879369.721315:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879369.721320:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879369.721324:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879369.721333:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879369.721377:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879369.721394:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100a680 x1875765290156416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879369.721409:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879369.721416:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879369.721446:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879369.721451:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879369.721460:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879369.721462:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879369.721464:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879369.721469:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879369.721473:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10308 00000100:00080000:1.0:1788879369.721478:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879369.721481:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879369.721483:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879369.721490:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879369.721492:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879369.721495:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879369.721499:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879369.721502:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879369.721506:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879369.721510:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879369.721515:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879369.721519:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879369.721528:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879369.721531:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879369.721533:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879369.721542:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:1.0:1788879369.721543:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879369.721557:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.721559:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879369.721565:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.721569:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879369.721576:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788879369.721577:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879369.721584:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879369.721586:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879369.721606:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879369.721610:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879369.721610:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879369.721614:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879369.721618:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879369.721626:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879369.721626:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879369.721676:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b800 x1875765290156544/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879369.721677:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290156416:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879369.721707:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879369.721719:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879369.721727:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879369.721740:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788879369.721752:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879369.721755:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879369.721759:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879369.721763:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879369.721769:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879369.721774:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879369.721776:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879369.721779:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879369.721784:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879369.721787:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1788879369.721795:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10308 00000100:00000001:3.0:1788879369.721797:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286517888 : -117763423033728 : ffff94e51100a680) 00000100:00080000:1.0:1788879369.721806:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879369.721810:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879369.721812:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879369.721818:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879369.721825:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879369.721826:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879369.721829:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879369.721834:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879369.721837:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879369.721841:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879369.721845:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879369.721850:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879369.721852:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879369.721857:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879369.721861:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879369.721863:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879369.721868:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788879369.721881:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00000001:1.0:1788879369.721883:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879369.721889:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879369.721897:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879369.721899:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.721903:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879369.721967:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879369.721971:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.721974:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879369.721982:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.721988:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304981546 00000100:00000001:1.0:1788879369.721991:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879369.722000:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879369.722004:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.722010:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788879369.722015:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.722025:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304981546 00000100:00100000:1.0:1788879369.722027:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b480 x1875765290156672/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879369.722039:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879369.722048:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879369.722053:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879369.722055:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.722061:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879369.722065:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.722069:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1788879369.722072:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879369.722079:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879369.722083:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1788879369.722087:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879369/real 1788879369] req@ffff94e51100a680 x1875765290156416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1788879369.722088:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879369.722109:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879369.722126:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10308 00000400:00000001:3.0:1788879369.722130:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879369.722136:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:1.0:1788879369.722144:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879369.722145:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879369.722154:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879369.722158:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879369.722159:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879369.722172:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879369.722172:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879369.722176:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1788879369.722195:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:3.0:1788879369.722197:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100a680 x1875765290156416/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788879369.722207:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879369.722219:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879369.722221:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1788879369.722235:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879369.722237:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879369.722259:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879369.722261:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879369.722270:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879369.722272:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879369.722278:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788879369.722289:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879369.722291:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879369.722302:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879369.722304:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879369.722318:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:3.0:1788879369.722322:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:1.0:1788879369.722327:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879369.722334:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 02000000:00000001:1.0:1788879369.722337:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.722373:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879369.722382:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290156416:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879369.722388:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.722392:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879369.722394:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879369.722397:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879369.722403:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879369.722406:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879369.722409:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879369.722414:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879369.722416:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879369.722421:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879369.722422:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.722425:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.722434:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879369.722437:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.722440:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879369.722443:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879369.722445:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879369.722447:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879369.722450:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879369.722453:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879369.722456:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879369.722459:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879369.722463:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290156544:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879369.722468:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879369.722471:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879369.722473:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879369.722475:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879369.722479:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.722488:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879369.722489:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:3.0:1788879369.722495:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879369.722514:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879369.722519:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879369.722522:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879369.722524:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.722526:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879369.722530:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879369.722546:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879369.722548:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304981004 00000008:00100000:1.0:1788879369.722552:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879369.722556:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304981546 00000100:00000001:3.0:1788879369.722573:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304981004 00000008:00000001:1.0:1788879369.722575:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.722594:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304981546 00000100:00000001:1.0:1788879369.722596:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.722618:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879369.722622:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879369.722623:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879369.722629:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.722630:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1788879369.722643:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879369.722650:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:3.0:1788879369.722653:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879369/real 1788879369] req@ffff94e51100b800 x1875765290156544/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788879369.722657:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879369.722684:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1788879369.722685:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879369.722688:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879369.722702:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00100000:1.0:1788879369.722710:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100aa00 x1875765290156800/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879369.722711:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879369.722721:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.722739:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879369.722755:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879369.722755:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879369.722772:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879369.722775:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290156544/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879369.722784:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.722802:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.722808:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879369.722812:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.722818:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879369.722820:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.722823:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879369.722851:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879369.722856:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879369.722862:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879369.722865:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879369.722869:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290156544:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879369.722874:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879369.722876:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879369.722877:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879369.722880:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879369.722883:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879369.722886:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879369.722890:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290156672:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879369.722894:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879369.722897:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879369.722899:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879369.722901:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879369.722935:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.722943:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879369.722945:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:3.0:1788879369.722949:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879369.722963:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879369.722970:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879369.722973:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879369.722975:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.722978:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879369.722981:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.722985:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304981003 00000100:00000001:3.0:1788879369.722988:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304981546 00000100:00000001:3.0:1788879369.722991:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304981003 00000100:00000001:3.0:1788879369.722994:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304981546 00000100:00000001:3.0:1788879369.722997:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879369.723000:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.723002:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879369.723007:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879369/real 1788879369] req@ffff94e51100b480 x1875765290156672/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879369.723016:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879369.723018:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879369.723021:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879369.723023:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879369.723025:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.723027:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879369.723029:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879369.723033:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290156672/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879369.723041:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.723044:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879369.723046:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.723048:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879369.723050:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.723052:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879369.723066:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879369.723069:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879369.723073:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879369.723076:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879369.723080:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290156672:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879369.723085:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.723087:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879369.723088:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879369.723090:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879369.723093:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879369.723094:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879369.723096:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879369.723099:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879369.723101:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879369.723104:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879369.723106:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.723108:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879369.723109:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879369.723111:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879369.723113:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879369.723114:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879369.723116:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879369.723118:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879369.723120:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879369.723121:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879369.723123:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.723125:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.723128:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879369.723130:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.723133:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879369.723134:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879369.723136:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879369.723138:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879369.723139:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879369.723142:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879369.723144:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879369.723147:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879369.723151:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290156800:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879369.723155:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879369.723158:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879369.723160:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879369.723162:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879369.723165:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.723171:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879369.723173:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:3.0:1788879369.723177:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879369.723189:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879369.723193:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879369.723196:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879369.723197:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.723200:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879369.723203:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.723207:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304981003 00000100:00000001:3.0:1788879369.723210:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304981546 00000100:00000001:3.0:1788879369.723213:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304981003 00000100:00000001:3.0:1788879369.723215:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304981546 00000100:00000001:3.0:1788879369.723218:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879369.723220:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.723222:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879369.723227:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879369/real 1788879369] req@ffff94e51100aa00 x1875765290156800/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879369.723236:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879369.723237:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879369.723240:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879369.723242:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879369.723244:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.723246:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879369.723247:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879369.723252:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290156800/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879369.723260:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.723262:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879369.723265:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.723267:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879369.723269:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.723271:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879369.723285:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879369.723288:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879369.723292:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879369.723295:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879369.723299:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290156800:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879369.723303:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.723305:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879369.723306:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879369.723308:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879369.723310:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879369.723311:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879369.723312:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879369.723315:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879369.723316:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879369.723318:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879369.723320:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.723322:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879369.723325:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879369.723326:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879369.723329:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879369.723331:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879369.785050:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879369.785062:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879369.785066:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879369.785078:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879369.785085:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879369.785091:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879369.785096:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879369.785101:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879369.785107:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879370.489026:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879370.489034:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879370.489040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879370.489042:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879370.489065:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879370.489068:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879370.489071:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879370.809026:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879370.809036:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879370.809038:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879370.809045:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879370.809049:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879370.809055:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879370.809058:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879370.809060:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879370.809063:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879371.513097:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879371.513108:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879371.513118:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879371.513120:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879371.513184:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879371.513188:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879371.513192:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879371.833031:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879371.833043:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879371.833045:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879371.833052:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879371.833057:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879371.833063:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879371.833066:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879371.833070:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879371.833073:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879372.537117:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879372.537128:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879372.537152:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879372.537155:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879372.537190:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879372.537195:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879372.537200:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879372.857050:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879372.857062:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879372.857065:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879372.857073:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879372.857079:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879372.857084:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879372.857088:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879372.857092:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879372.857097:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879373.560972:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879373.560979:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879373.560985:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879373.561005:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879373.561008:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879373.561014:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879373.561019:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879373.561023:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879373.561027:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879373.561031:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879373.561142:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879373.561148:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879373.561153:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879373.881073:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879373.881081:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879373.881083:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879373.881089:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879373.881092:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879373.881097:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879373.881100:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879373.881103:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879373.881105:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879374.585039:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879374.585046:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879374.585051:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879374.585053:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879374.585076:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879374.585078:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879374.585081:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879374.840998:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879374.841010:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879374.841013:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879374.841016:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879374.841022:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879374.841029:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10313 00000100:00080000:2.0:1788879374.841041:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879374.841045:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879374.841047:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879374.841055:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879374.841058:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879374.841061:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879374.841074:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879374.841076:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879374.841078:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879374.841082:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879374.841085:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879374.841089:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879374.841092:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879374.841095:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879374.841097:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879374.841100:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879374.841113:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879374.841114:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879374.841117:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879374.841120:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879374.841125:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879374.841139:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879374.841148:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290156928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879374.841159:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879374.841163:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879374.841193:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879374.841196:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879374.841199:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879374.841200:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879374.841202:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879374.841204:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879374.841206:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10313 00000100:00080000:2.0:1788879374.841210:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879374.841211:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879374.841213:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879374.841216:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879374.841217:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879374.841219:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879374.841221:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879374.841223:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879374.841224:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879374.841227:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879374.841229:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879374.841230:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879374.841235:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879374.841235:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879374.841238:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879374.841241:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879374.841243:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1788879374.841244:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879374.841248:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00000001:2.0:1788879374.841249:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879374.841252:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.841254:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788879374.841257:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879374.841258:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879374.841259:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879374.841264:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879374.841266:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879374.841267:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879374.841268:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879374.841273:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879374.841275:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879374.841280:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879374.841282:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4000 x1875765290157056/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879374.841289:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290156928:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879374.841291:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879374.841295:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879374.841299:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.841301:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1788879374.841303:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1788879374.841319:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879374.841323:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879374.841326:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879374.841339:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.841355:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879374.841357:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00000001:3.0:1788879374.841361:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879374.841397:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879374.841408:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879374.841411:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879374.841413:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.841415:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879374.841419:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.841422:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304986665 00000100:00000001:3.0:1788879374.841431:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879374.841443:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304986665 00000100:00000001:3.0:1788879374.841447:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879374.841449:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879374.841451:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.841453:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879374.841458:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879374/real 1788879374] req@ffff94e5045e7b80 x1875765290156928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879374.841468:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879374.841487:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879374.841491:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879374.841494:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879374.841497:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.841499:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879374.841510:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879374.841517:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290156928/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879374.841521:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879374.841527:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879374.841532:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879374.841534:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879374.841543:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879374.841543:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879374.841550:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879374.841553:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10313 00000100:00000001:3.0:1788879374.841554:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879374.841557:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.841563:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879374.841565:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879374.841567:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1788879374.841569:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788879374.841576:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00080000:3.0:1788879374.841579:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879374.841581:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879374.841583:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879374.841593:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290156928:192.168.203.129@tcp:250:kworker.0 00000020:00000001:2.0:1788879374.841594:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879374.841599:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879374.841601:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788879374.841603:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879374.841606:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1788879374.841608:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879374.841610:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879374.841615:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788879374.841616:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879374.841620:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:3.0:1788879374.841621:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879374.841625:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00080000:2.0:1788879374.841626:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000020:00000001:3.0:1788879374.841629:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:2.0:1788879374.841631:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879374.841635:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879374.841635:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879374.841639:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1788879374.841640:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879374.841642:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 02000000:00000001:2.0:1788879374.841645:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.841646:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879374.841649:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879374.841651:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1788879374.841654:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879374.841658:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879374.841660:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.841661:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879374.841665:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879374.841666:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879374.841669:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879374.841670:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879374.841675:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879374.841675:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879374.841681:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788879374.841682:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879374.841688:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788879374.841689:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.841691:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879374.841695:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879374.841698:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879374.841699:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879374.841704:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879374.841708:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290157056:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879374.841712:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879374.841714:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879374.841717:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879374.841719:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1788879374.841724:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290157184/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879374.841725:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.841736:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879374.841740:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074548224 : -117763635003392 : ffff94e5045e4000) 00000100:00000001:3.0:1788879374.841750:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879374.841753:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879374.841757:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879374.841761:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879374.841764:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879374.841774:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000800:00000100:3.0:1788879374.841779:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879374.841780:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879374.841785:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879374.841789:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879374.841790:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879374.841796:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788879374.841797:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10313 00000100:00000001:3.0:1788879374.841800:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.841804:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:2.0:1788879374.841805:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879374.841808:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879374.841810:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879374.841814:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879374.841817:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304986123 00000020:00000001:2.0:1788879374.841820:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879374.841823:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879374.841826:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304986665 00000020:00000001:2.0:1788879374.841827:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879374.841835:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879374.841835:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304986123 00000100:00000001:2.0:1788879374.841843:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879374.841844:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304986665 00000100:00000001:2.0:1788879374.841846:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879374.841850:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788879374.841854:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879374.841856:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879374.841860:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.841861:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1788879374.841865:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879374.841885:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788879374.841887:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879374/real 1788879374] req@ffff94e5045e4000 x1875765290157056/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879374.841888:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879374.841892:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879374.841895:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879374.841899:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879374.841903:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879374.841903:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879374.841906:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1788879374.841908:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879374.841909:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:2.0:1788879374.841911:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.841913:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879374.841960:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879374.841962:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879374.841965:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879374.841972:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879374.841980:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290157312/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879374.841981:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.841989:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879374.841993:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879374.841997:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879374.842002:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879374.842004:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4000 x1875765290157056/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879374.842006:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.842019:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842023:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879374.842025:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842027:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879374.842030:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842032:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879374.842048:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879374.842051:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879374.842054:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879374.842056:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879374.842059:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290157056:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879374.842063:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842066:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879374.842067:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879374.842070:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879374.842073:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879374.842075:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879374.842076:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879374.842078:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879374.842080:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879374.842083:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879374.842084:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842086:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842090:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879374.842092:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.842095:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879374.842096:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879374.842098:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879374.842099:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879374.842101:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879374.842103:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879374.842105:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879374.842107:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879374.842112:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290157184:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879374.842115:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879374.842118:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879374.842120:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879374.842122:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879374.842127:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.842135:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879374.842138:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00000001:3.0:1788879374.842142:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879374.842158:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879374.842162:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879374.842164:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879374.842166:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.842168:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879374.842188:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.842191:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304986122 00000100:00000001:3.0:1788879374.842193:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304986665 00000100:00000001:3.0:1788879374.842195:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304986122 00000100:00000001:3.0:1788879374.842197:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304986665 00000100:00000001:3.0:1788879374.842199:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879374.842200:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.842202:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879374.842206:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879374/real 1788879374] req@ffff94e5045e7b80 x1875765290157184/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879374.842212:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879374.842214:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879374.842217:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879374.842219:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879374.842221:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842222:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879374.842223:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879374.842226:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290157184/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879374.842231:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842233:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879374.842235:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842236:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879374.842238:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842240:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879374.842257:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879374.842260:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879374.842262:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879374.842265:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879374.842268:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290157184:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879374.842271:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879374.842271:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879374.842272:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879374.842274:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879374.842275:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879374.842277:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879374.842279:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290157312:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879374.842281:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879374.842282:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879374.842284:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879374.842285:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879374.842288:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.842292:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879374.842292:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:3.0:1788879374.842295:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879374.842303:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879374.842320:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879374.842390:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879374.842393:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.842395:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879374.842399:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.842402:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304986122 00000100:00000001:3.0:1788879374.842404:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304986665 00000100:00000001:3.0:1788879374.842407:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304986122 00000100:00000001:3.0:1788879374.842409:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304986665 00000100:00000001:3.0:1788879374.842413:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879374.842414:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.842416:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879374.842420:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879374/real 1788879374] req@ffff94e5045e4e00 x1875765290157312/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879374.842427:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879374.842428:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879374.842432:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879374.842435:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879374.842437:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842439:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879374.842441:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879374.842445:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290157312/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879374.842450:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842452:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879374.842454:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842455:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879374.842456:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842457:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879374.842487:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879374.842489:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879374.842492:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879374.842495:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879374.842498:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290157312:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879374.842503:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842507:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879374.842509:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879374.842511:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879374.842514:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879374.842516:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879374.842519:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879374.842522:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879374.842524:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879374.842526:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879374.842527:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842529:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879374.842530:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879374.842531:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879374.842533:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879374.842535:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879374.842537:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879374.842540:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879374.842541:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879374.842544:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879374.842545:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842548:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879374.842552:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879374.842554:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879374.842556:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879374.842558:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879374.904989:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879374.904995:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879374.904997:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879374.905001:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879374.905004:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879374.905007:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879374.905010:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879374.905012:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879374.905030:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879375.609044:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879375.609052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879375.609058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879375.609060:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879375.609086:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879375.609089:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879375.609093:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879375.929103:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879375.929112:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879375.929115:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879375.929120:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879375.929125:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879375.929130:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879375.929133:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879375.929136:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879375.929139:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879376.633009:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879376.633018:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879376.633021:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879376.633025:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879376.633031:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879376.633034:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879376.633036:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879376.633039:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879376.633043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879376.633045:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879376.633079:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879376.633082:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879376.633086:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879376.953007:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879376.953016:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879376.953018:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879376.953025:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879376.953030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879376.953035:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879376.953038:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879376.953052:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879376.953056:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879377.657024:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879377.657031:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879377.657038:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879377.657050:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879377.657218:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879377.657223:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879377.657228:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879377.976963:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879377.976969:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879377.976971:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879377.976976:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879377.976978:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879377.976981:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879377.976984:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879377.976986:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879377.976987:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879378.680983:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879378.680988:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879378.680992:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879378.680993:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879378.681011:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879378.681013:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879378.681015:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879379.001014:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879379.001022:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879379.001024:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879379.001029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879379.001034:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879379.001037:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.001040:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879379.001042:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.001045:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879379.705005:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879379.705012:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879379.705016:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879379.705018:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879379.705045:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879379.705049:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879379.705053:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879379.961060:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879379.961077:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879379.961081:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879379.961086:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879379.961092:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879379.961102:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10318 00000100:00080000:3.0:1788879379.961117:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879379.961121:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879379.961124:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879379.961135:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879379.961139:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879379.961143:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879379.961147:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879379.961149:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879379.961150:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879379.961156:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879379.961161:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879379.961166:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879379.961171:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879379.961175:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879379.961178:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879379.961182:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879379.961198:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879379.961201:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879379.961205:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879379.961210:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879379.961217:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879379.961234:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879379.961248:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290157440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879379.961261:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879379.961268:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879379.961306:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879379.961310:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879379.961314:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879379.961316:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879379.961318:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879379.961322:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879379.961325:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10318 00000100:00080000:3.0:1788879379.961330:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879379.961332:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879379.961334:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879379.961338:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879379.961341:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879379.961345:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879379.961347:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879379.961350:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879379.961352:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879379.961358:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879379.961362:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879379.961365:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879379.961373:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879379.961376:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879379.961379:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879379.961386:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879379.961387:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:3.0:1788879379.961391:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879379.961400:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.961404:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788879379.961408:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879379.961409:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788879379.961415:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.961417:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879379.961421:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879379.961424:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879379.961432:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879379.961434:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879379.961437:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879379.961441:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879379.961447:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879379.961450:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879379.961460:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290157440:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879379.961470:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1788879379.961473:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290157568/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879379.961477:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879379.961483:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879379.961493:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879379.961501:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879379.961504:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879379.961506:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879379.961512:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879379.961519:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879379.961524:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879379.961525:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879379.961528:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879379.961529:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879379.961536:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:3.0:1788879379.961538:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879379.961544:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879379.961550:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10318 00000100:00080000:3.0:1788879379.961558:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879379.961561:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879379.961562:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879379.961567:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879379.961569:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879379.961573:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879379.961575:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879379.961578:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879379.961582:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879379.961587:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879379.961591:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879379.961594:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:1.0:1788879379.961601:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879379.961601:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879379.961611:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879379.961614:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879379.961620:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879379.961621:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00000001:3.0:1788879379.961625:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.961630:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879379.961632:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.961634:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.961640:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879379.961646:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879379.961647:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879379.961653:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.961656:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304991785 00000100:00000001:3.0:1788879379.961661:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879379.961667:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879379.961670:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879379.961679:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.961687:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00100000:3.0:1788879379.961689:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290157696/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879379.961703:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879379.961705:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879379.961709:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.961710:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304991785 00000100:00080000:3.0:1788879379.961717:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879379.961719:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879379.961727:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879379.961731:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879379.961732:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879379.961740:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879379.961741:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788879379.961750:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788879379.961758:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879379/real 1788879379] req@ffff94e50449ce00 x1875765290157440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879379.961760:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879379.961777:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10318 00000400:00000001:1.0:1788879379.961794:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1788879379.961796:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000400:00000001:1.0:1788879379.961798:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879379.961807:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879379.961808:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879379.961817:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879379.961818:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879379.961821:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.961827:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879379.961831:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788879379.961832:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:1.0:1788879379.961840:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290157440/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788879379.961849:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879379.961862:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788879379.961863:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879379.961874:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1788879379.961877:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879379.961880:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.961890:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879379.961893:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879379.961893:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879379.961949:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879379.961961:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879379.961965:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879379.961967:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879379.961974:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879379.961978:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879379.961980:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879379.961983:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879379.961987:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879379.961990:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.961997:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:3.0:1788879379.962000:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879379.962005:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788879379.962006:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879379.962014:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879379.962019:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00080000:1.0:1788879379.962020:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1788879379.962022:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879379.962034:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879379.962037:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.962039:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879379.962061:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290157440:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1788879379.962062:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290157824/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879379.962076:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962084:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879379.962090:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879379.962101:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788879379.962102:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879379.962109:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1788879379.962110:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879379.962113:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879379.962120:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788879379.962121:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879379.962129:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879379.962133:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879379.962138:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879379.962140:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962144:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962153:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879379.962157:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.962161:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879379.962163:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879379.962166:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879379.962168:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879379.962170:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879379.962174:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879379.962190:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879379.962193:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879379.962198:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290157568:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879379.962203:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879379.962207:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879379.962209:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879379.962211:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879379.962215:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.962226:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879379.962228:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000100:00000001:1.0:1788879379.962234:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879379.962254:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879379.962261:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879379.962264:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879379.962266:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.962268:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879379.962299:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.962306:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304991244 00000100:00000001:1.0:1788879379.962313:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304991785 00000100:00000001:1.0:1788879379.962317:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304991244 00000100:00000001:1.0:1788879379.962322:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304991785 00000100:00000001:1.0:1788879379.962327:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879379.962331:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.962335:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879379.962345:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879379/real 1788879379] req@ffff94e50449e680 x1875765290157568/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879379.962365:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879379.962367:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879379.962374:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879379.962377:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879379.962380:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962384:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879379.962387:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879379.962396:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290157568/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879379.962411:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962414:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879379.962417:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962420:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879379.962422:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962424:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879379.962439:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879379.962442:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879379.962447:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879379.962450:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879379.962455:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290157568:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879379.962460:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879379.962462:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879379.962464:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879379.962468:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879379.962470:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879379.962473:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879379.962477:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290157696:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879379.962481:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879379.962485:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879379.962487:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879379.962489:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879379.962494:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.962503:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879379.962504:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:1.0:1788879379.962510:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879379.962527:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879379.962533:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879379.962536:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879379.962539:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.962541:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879379.962545:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.962549:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304991242 00000100:00000001:1.0:1788879379.962552:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304991785 00000100:00000001:1.0:1788879379.962555:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304991242 00000100:00000001:1.0:1788879379.962558:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304991785 00000100:00000001:1.0:1788879379.962561:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879379.962563:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.962566:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879379.962571:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879379/real 1788879379] req@ffff94e50449df80 x1875765290157696/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879379.962580:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879379.962582:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879379.962585:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879379.962587:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879379.962589:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962591:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879379.962593:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879379.962597:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290157696/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879379.962606:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962608:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879379.962611:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962613:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879379.962615:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962617:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879379.962629:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879379.962632:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879379.962635:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879379.962637:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879379.962642:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290157696:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879379.962647:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879379.962649:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879379.962651:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879379.962654:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879379.962656:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879379.962658:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879379.962662:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290157824:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879379.962666:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879379.962668:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879379.962670:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879379.962672:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879379.962675:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.962681:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879379.962682:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:1.0:1788879379.962686:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879379.962698:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879379.962703:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879379.962705:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879379.962707:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.962709:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879379.962712:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.962716:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304991242 00000100:00000001:1.0:1788879379.962719:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304991785 00000100:00000001:1.0:1788879379.962722:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304991242 00000100:00000001:1.0:1788879379.962725:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304991785 00000100:00000001:1.0:1788879379.962727:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879379.962730:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.962732:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879379.962736:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879379/real 1788879379] req@ffff94e50449d180 x1875765290157824/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879379.962754:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879379.962756:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879379.962758:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879379.962760:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879379.962762:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962764:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879379.962766:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879379.962770:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290157824/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879379.962778:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962780:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879379.962783:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962784:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879379.962786:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962788:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879379.962801:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879379.962804:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879379.962808:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879379.962811:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879379.962815:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290157824:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879379.962829:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962831:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879379.962833:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879379.962835:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879379.962838:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879379.962840:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879379.962842:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879379.962845:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879379.962846:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879379.962849:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879379.962851:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962854:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879379.962855:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879379.962857:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879379.962860:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879379.962861:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879379.962863:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879379.962865:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879379.962866:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879379.962868:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879379.962870:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962871:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879379.962873:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879379.962874:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879379.962876:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879379.962877:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879379.962878:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879379.962880:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879379.962881:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879379.962883:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879379.962884:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962887:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879379.962890:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879379.962892:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879379.962895:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879379.962897:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879380.025035:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879380.025042:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879380.025045:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879380.025050:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879380.025054:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879380.025058:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879380.025061:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879380.025064:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879380.025082:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879380.728976:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879380.728983:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879380.728988:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879380.728989:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879380.729009:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879380.729011:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879380.729014:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879381.048998:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879381.049010:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879381.049014:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879381.049023:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879381.049029:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879381.049035:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879381.049040:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879381.049045:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879381.049050:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879381.753130:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879381.753141:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879381.753148:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879381.753150:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879381.753277:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879381.753293:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879381.753299:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879382.073051:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879382.073060:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879382.073063:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879382.073071:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879382.073076:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879382.073082:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879382.073086:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879382.073090:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879382.073094:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879382.777095:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879382.777103:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879382.777109:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879382.777111:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879382.777189:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879382.777193:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879382.777198:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879383.097035:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879383.097052:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879383.097055:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879383.097064:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879383.097069:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879383.097075:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879383.097080:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879383.097084:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879383.097088:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879383.801101:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879383.801112:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879383.801122:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879383.801128:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879383.801142:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879383.801148:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879383.801153:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879383.801157:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879383.801162:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879383.801166:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879383.801230:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879383.801235:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879383.801240:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879384.121121:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879384.121135:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879384.121139:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879384.121166:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879384.121174:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879384.121182:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879384.121187:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879384.121193:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879384.121199:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879384.825047:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879384.825055:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879384.825062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879384.825064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879384.825136:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879384.825139:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879384.825143:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879385.081064:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879385.081076:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879385.081079:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879385.081083:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879385.081089:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879385.081098:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10323 00000100:00080000:3.0:1788879385.081113:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879385.081117:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879385.081121:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879385.081132:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879385.081136:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879385.081142:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879385.081146:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879385.081150:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879385.081152:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879385.081160:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879385.081165:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879385.081170:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879385.081176:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879385.081181:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879385.081185:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879385.081189:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879385.081215:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879385.081217:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879385.081221:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879385.081224:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879385.081229:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879385.081265:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879385.081281:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290157952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879385.081300:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879385.081307:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879385.081338:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879385.081344:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879385.081349:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879385.081351:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879385.081354:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879385.081358:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879385.081362:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10323 00000100:00080000:3.0:1788879385.081368:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879385.081371:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879385.081373:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879385.081379:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879385.081381:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879385.081384:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879385.081387:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879385.081390:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879385.081392:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879385.081398:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1788879385.081399:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879385.081406:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.081407:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788879385.081410:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879385.081413:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879385.081417:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788879385.081419:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879385.081420:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879385.081431:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879385.081433:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879385.081434:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879385.081440:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1788879385.081443:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879385.081446:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788879385.081450:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879385.081455:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879385.081457:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290157952:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788879385.081462:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879385.081466:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879385.081468:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879385.081474:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788879385.081474:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879385.081478:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879385.081480:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879385.081484:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879385.081486:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879385.081496:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879385.081500:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.081517:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879385.081523:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00100000:3.0:1788879385.081524:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290158080/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879385.081531:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879385.081541:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879385.081545:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879385.081552:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879385.081556:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879385.081560:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879385.081562:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879385.081565:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879385.081568:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879385.081570:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10323 00000100:00080000:3.0:1788879385.081575:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788879385.081582:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879385.081583:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879385.081590:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879385.081595:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879385.081597:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879385.081600:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879385.081602:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:3.0:1788879385.081603:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879385.081609:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879385.081611:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879385.081612:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879385.081618:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879385.081623:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879385.081624:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879385.081631:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.081634:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879385.081635:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879385.081643:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879385.081647:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879385.081649:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879385.081651:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879385.081655:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.081657:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4304996905 00000100:00000001:3.0:1788879385.081658:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879385.081670:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879385.081674:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879385.081676:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879385.081678:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788879385.081679:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879385.081685:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879385.081694:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.081700:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4304996905 00000100:00100000:3.0:1788879385.081701:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290158208/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879385.081714:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879385.081722:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879385.081730:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.081738:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879385.081739:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879385.081745:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879385.081751:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879385.081752:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879385/real 1788879385] req@ffff94e50449c380 x1875765290157952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879385.081755:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879385.081771:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879385.081777:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879385.081782:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:1.0:1788879385.081784:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788879385.081788:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000400:00000001:1.0:1788879385.081789:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879385.081798:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:3.0:1788879385.081799:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10323 00000100:00000001:1.0:1788879385.081805:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00080000:3.0:1788879385.081807:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879385.081810:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879385.081812:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879385.081815:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879385.081818:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879385.081826:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290157952/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879385.081827:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879385.081836:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879385.081840:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879385.081845:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879385.081853:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879385.081855:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879385.081860:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879385.081864:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879385.081865:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879385.081872:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.081876:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879385.081880:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788879385.081882:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879385.081884:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879385.081890:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879385.081893:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879385.081943:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1788879385.081954:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:1.0:1788879385.081956:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 02000000:00000001:3.0:1788879385.081964:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879385.081966:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1788879385.081971:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879385.081976:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879385.081982:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879385.081983:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290157952:192.168.203.129@tcp:250:kworker.0 00000008:00000001:3.0:1788879385.081987:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879385.081995:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.081999:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879385.082004:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879385.082005:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788879385.082009:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.082011:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879385.082017:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788879385.082018:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879385.082023:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879385.082025:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:3.0:1788879385.082027:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879385.082032:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879385.082036:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788879385.082038:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879385.082041:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879385.082045:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879385.082048:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00100000:3.0:1788879385.082051:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290158336/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879385.082052:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082059:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082072:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879385.082073:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879385.082076:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879385.082079:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.082083:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879385.082086:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.082088:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879385.082093:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879385.082096:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879385.082098:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879385.082102:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879385.082104:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879385.082107:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879385.082113:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290158080:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879385.082117:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879385.082120:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879385.082122:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879385.082124:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879385.082129:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.082138:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879385.082139:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:1.0:1788879385.082145:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879385.082163:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879385.082169:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879385.082172:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879385.082173:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.082175:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879385.082178:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.082183:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304996364 00000100:00000001:1.0:1788879385.082186:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304996905 00000100:00000001:1.0:1788879385.082189:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304996364 00000100:00000001:1.0:1788879385.082191:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304996905 00000100:00000001:1.0:1788879385.082194:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879385.082198:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.082200:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879385.082208:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879385/real 1788879385] req@ffff94e50449fb80 x1875765290158080/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879385.082223:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879385.082226:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879385.082230:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879385.082233:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879385.082257:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082261:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879385.082263:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879385.082271:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290158080/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879385.082296:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082300:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879385.082305:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082307:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879385.082309:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082311:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879385.082324:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879385.082327:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879385.082331:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879385.082334:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879385.082338:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290158080:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879385.082343:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879385.082345:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879385.082347:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879385.082349:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879385.082352:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879385.082354:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879385.082358:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290158208:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879385.082361:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879385.082363:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879385.082365:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879385.082367:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879385.082371:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.082379:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879385.082380:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:1.0:1788879385.082385:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879385.082400:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879385.082406:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879385.082409:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879385.082410:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.082413:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879385.082416:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.082420:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304996362 00000100:00000001:1.0:1788879385.082423:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304996905 00000100:00000001:1.0:1788879385.082425:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304996362 00000100:00000001:1.0:1788879385.082427:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304996905 00000100:00000001:1.0:1788879385.082430:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879385.082432:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.082434:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879385.082438:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879385/real 1788879385] req@ffff94e50449d880 x1875765290158208/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879385.082447:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879385.082448:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879385.082451:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879385.082452:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879385.082454:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082456:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879385.082457:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879385.082461:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290158208/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879385.082469:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082472:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879385.082474:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082476:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879385.082477:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082479:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879385.082490:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879385.082493:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879385.082498:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879385.082502:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879385.082508:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290158208:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879385.082514:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879385.082516:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879385.082518:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879385.082522:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879385.082525:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879385.082528:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879385.082534:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290158336:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879385.082540:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879385.082542:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879385.082545:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879385.082548:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879385.082553:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.082562:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879385.082564:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:1.0:1788879385.082570:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879385.082590:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879385.082596:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879385.082599:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879385.082601:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.082603:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879385.082606:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.082610:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4304996362 00000100:00000001:1.0:1788879385.082613:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4304996905 00000100:00000001:1.0:1788879385.082615:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4304996362 00000100:00000001:1.0:1788879385.082618:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4304996905 00000100:00000001:1.0:1788879385.082620:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879385.082622:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.082624:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879385.082629:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879385/real 1788879385] req@ffff94e50449f800 x1875765290158336/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879385.082644:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879385.082646:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879385.082649:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879385.082650:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879385.082652:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082654:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879385.082655:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879385.082659:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290158336/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879385.082667:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082669:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879385.082671:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082673:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879385.082674:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082676:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879385.082687:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879385.082689:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879385.082692:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879385.082694:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879385.082698:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290158336:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879385.082701:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082704:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879385.082706:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879385.082708:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879385.082710:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879385.082712:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879385.082713:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879385.082716:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879385.082717:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879385.082720:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879385.082722:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082724:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879385.082726:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879385.082727:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879385.082730:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879385.082731:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879385.082732:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879385.082734:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879385.082735:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879385.082737:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879385.082738:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082740:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879385.082741:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879385.082742:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879385.082743:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879385.082745:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879385.082746:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879385.082747:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879385.082748:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879385.082750:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879385.082751:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082753:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.082757:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879385.082758:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.082760:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879385.082762:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879385.145061:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879385.145082:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879385.145085:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879385.145094:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879385.145100:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879385.145106:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.145110:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879385.145114:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879385.145119:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879385.849126:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879385.849136:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879385.849156:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879385.849159:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879385.849273:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879385.849280:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879385.849285:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879386.169029:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879386.169038:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879386.169040:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879386.169048:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879386.169053:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879386.169059:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879386.169062:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879386.169095:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879386.169099:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879386.873110:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879386.873123:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879386.873128:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879386.873133:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879386.873141:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879386.873144:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879386.873147:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879386.873150:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879386.873156:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879386.873158:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879386.873285:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879386.873289:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879386.873293:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879387.193021:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879387.193031:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879387.193034:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879387.193040:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879387.193045:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879387.193050:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879387.193053:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879387.193057:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879387.193060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879387.896988:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879387.896994:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879387.896999:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879387.897000:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879387.897027:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879387.897029:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879387.897032:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879388.217042:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879388.217050:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879388.217053:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879388.217059:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879388.217064:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879388.217068:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879388.217071:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879388.217074:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879388.217078:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879388.921059:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879388.921069:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879388.921079:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879388.921082:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879388.921162:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879388.921167:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879388.921171:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879389.241017:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879389.241025:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879389.241028:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879389.241035:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879389.241040:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879389.241044:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879389.241048:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879389.241051:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879389.241055:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879389.945041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879389.945050:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879389.945056:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879389.945059:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879389.945184:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879389.945191:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879389.945196:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879390.201083:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879390.201096:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879390.201100:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879390.201114:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879390.201121:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879390.201129:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10329 00000100:00080000:2.0:1788879390.201143:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879390.201158:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879390.201160:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879390.201171:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879390.201174:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879390.201178:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879390.201182:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879390.201184:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879390.201186:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879390.201192:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879390.201195:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879390.201221:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879390.201227:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879390.201231:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879390.201236:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879390.201240:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879390.201257:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879390.201262:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879390.201267:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879390.201271:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879390.201279:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879390.201298:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879390.201314:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290158464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879390.201336:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879390.201342:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879390.201362:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879390.201367:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879390.201372:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879390.201374:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879390.201377:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879390.201381:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879390.201386:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10329 00000100:00080000:2.0:1788879390.201394:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879390.201398:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879390.201400:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879390.201406:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879390.201408:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879390.201410:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788879390.201412:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879390.201418:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788879390.201420:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879390.201426:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879390.201427:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879390.201433:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879390.201436:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788879390.201445:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879390.201446:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879390.201457:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879390.201458:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879390.201466:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879390.201467:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879390.201473:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879390.201475:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788879390.201486:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290158464:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879390.201487:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879390.201506:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879390.201512:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:2.0:1788879390.201516:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879390.201520:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879390.201524:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879390.201527:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:2.0:1788879390.201535:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879390.201536:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879390.201542:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879390.201553:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879390.201555:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879390.201557:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879390.201559:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879390.201562:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788879390.201564:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.201570:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074559872 : -117763634991744 : ffff94e5045e6d80) 00000100:00000001:1.0:1788879390.201580:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879390.201580:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879390.201595:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290158592/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879390.201613:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879390.201639:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879390.201641:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879390.201651:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879390.201660:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879390.201666:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788879390.201669:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879390.201676:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879390.201679:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788879390.201682:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879390.201687:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879390.201693:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879390.201696:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:2.0:1788879390.201716:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10329 00000100:00000001:1.0:1788879390.201721:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879390.201729:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879390.201732:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305002025 00000100:00000001:2.0:1788879390.201737:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879390.201744:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879390.201748:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1788879390.201752:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879390.201757:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879390.201760:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879390.201762:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305002025 00000020:00000001:2.0:1788879390.201765:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879390.201770:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879390.201771:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879390.201776:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879390.201780:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879390.201787:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879390.201791:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1788879390.201793:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879390.201801:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.201803:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879390.201805:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879390.201813:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788879390.201814:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879390.201822:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000008:00000001:2.0:1788879390.201823:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879390.201829:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879390.201837:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1788879390.201838:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879390/real 1788879390] req@ffff94e5045e6d80 x1875765290158464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879390.201841:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879390.201849:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:1.0:1788879390.201865:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1788879390.201867:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:1.0:1788879390.201870:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1788879390.201874:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.201879:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879390.201886:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.201887:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879390.201895:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.201943:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879390.201947:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879390.201956:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290158720/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879390.201958:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6d80 x1875765290158464/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879390.202005:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879390.202008:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879390.202013:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202017:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879390.202021:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202027:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879390.202030:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879390.202035:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1788879390.202039:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879390.202042:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879390.202046:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879390.202051:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1788879390.202052:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879390.202062:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879390.202063:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879390.202068:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10329 00000100:00080000:1.0:1788879390.202079:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879390.202080:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879390.202088:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879390.202094:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879390.202100:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00080000:1.0:1788879390.202102:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000020:00000001:2.0:1788879390.202103:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879390.202109:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879390.202118:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:2.0:1788879390.202118:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879390.202128:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879390.202133:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:1.0:1788879390.202140:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290158464:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1788879390.202142:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879390.202154:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879390.202158:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879390.202160:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879390.202163:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879390.202166:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879390.202173:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879390.202174:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879390.202177:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788879390.202180:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879390.202186:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000008:00100000:2.0:1788879390.202187:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000020:00000001:1.0:1788879390.202193:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000008:00000001:2.0:1788879390.202195:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879390.202221:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:2.0:1788879390.202223:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879390.202232:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879390.202236:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879390.202238:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1788879390.202244:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202246:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879390.202252:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879390.202256:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879390.202258:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879390.202265:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879390.202271:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1788879390.202273:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202277:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202284:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879390.202286:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879390.202289:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879390.202292:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879390.202292:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202309:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879390.202310:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290158848/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879390.202318:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879390.202324:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879390.202335:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879390.202351:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290158592:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879390.202351:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879390.202360:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202364:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788879390.202372:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879390.202373:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879390.202380:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879390.202385:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879390.202390:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202401:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879390.202404:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00000001:1.0:1788879390.202411:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879390.202433:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879390.202439:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879390.202443:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879390.202445:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202447:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879390.202450:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202456:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305001484 00000100:00000001:1.0:1788879390.202460:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305002025 00000100:00000001:1.0:1788879390.202463:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305001484 00000100:00000001:1.0:1788879390.202466:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305002025 00000100:00000001:1.0:1788879390.202469:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879390.202471:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202474:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879390.202479:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879390/real 1788879390] req@ffff94e5045e7480 x1875765290158592/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879390.202491:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879390.202492:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879390.202495:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879390.202497:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879390.202499:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.202502:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879390.202503:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879390.202509:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290158592/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879390.202519:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.202522:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879390.202525:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.202527:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879390.202529:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.202531:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879390.202546:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879390.202549:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879390.202553:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879390.202556:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879390.202560:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290158592:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879390.202565:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879390.202567:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879390.202569:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879390.202572:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879390.202574:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879390.202577:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879390.202580:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290158720:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879390.202584:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879390.202586:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879390.202588:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879390.202590:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879390.202594:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202599:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879390.202601:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:1.0:1788879390.202605:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879390.202623:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879390.202631:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879390.202636:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879390.202639:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202643:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879390.202648:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202655:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305001482 00000100:00000001:1.0:1788879390.202660:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305002025 00000100:00000001:1.0:1788879390.202664:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305001482 00000100:00000001:1.0:1788879390.202669:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305002025 00000100:00000001:1.0:1788879390.202673:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879390.202677:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202681:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879390.202690:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879390/real 1788879390] req@ffff94e5045e4700 x1875765290158720/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879390.202709:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879390.202711:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879390.202717:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879390.202720:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879390.202722:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.202724:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879390.202726:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879390.202732:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290158720/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879390.202741:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.202744:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879390.202747:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.202749:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879390.202751:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.202753:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879390.202765:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879390.202768:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879390.202772:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879390.202775:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879390.202780:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290158720:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879390.202785:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.202788:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879390.202789:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879390.202792:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879390.202795:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879390.202797:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879390.202799:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879390.202802:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879390.202803:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879390.202806:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879390.202807:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.202810:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879390.202811:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879390.202813:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879390.202815:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879390.202817:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879390.202819:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879390.202821:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879390.202822:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879390.202824:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879390.202825:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.202827:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.202831:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879390.202833:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202836:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879390.202838:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879390.202839:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879390.202841:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879390.202843:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879390.202845:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879390.202848:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879390.202850:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879390.202854:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290158848:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879390.202858:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879390.202860:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879390.202862:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879390.202864:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879390.202869:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202876:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879390.202878:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:1.0:1788879390.202883:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879390.202900:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879390.202936:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879390.202939:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879390.202941:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202943:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879390.202947:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202952:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305001483 00000100:00000001:1.0:1788879390.202955:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305002025 00000100:00000001:1.0:1788879390.202958:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305001483 00000100:00000001:1.0:1788879390.202961:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305002025 00000100:00000001:1.0:1788879390.202964:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879390.202966:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.202968:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879390.202974:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879390/real 1788879390] req@ffff94e5045e6680 x1875765290158848/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879390.202983:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879390.202985:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879390.202988:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879390.202990:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879390.202991:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.202994:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879390.202996:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879390.203000:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290158848/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879390.203009:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.203012:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879390.203015:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.203017:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879390.203019:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.203021:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879390.203034:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879390.203037:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879390.203041:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879390.203045:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879390.203051:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290158848:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879390.203059:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.203062:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879390.203065:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879390.203068:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879390.203072:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879390.203074:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879390.203076:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879390.203080:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879390.203092:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879390.203095:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879390.203097:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.203102:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.203107:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879390.203110:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.203114:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879390.203117:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879390.265004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879390.265011:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879390.265013:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879390.265018:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879390.265023:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879390.265026:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.265029:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879390.265032:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879390.265034:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879390.969042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879390.969052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879390.969062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879390.969066:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879390.969101:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879390.969106:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879390.969111:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879391.289012:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879391.289020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879391.289023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879391.289043:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879391.289052:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879391.289057:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879391.289063:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879391.289068:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879391.289074:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879391.992997:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879391.993003:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879391.993009:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879391.993011:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879391.993032:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879391.993034:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879391.993036:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879392.313007:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879392.313016:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879392.313018:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879392.313025:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879392.313030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879392.313034:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879392.313038:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879392.313041:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879392.313045:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879393.016994:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879393.016999:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879393.017003:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879393.017004:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879393.017021:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879393.017023:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879393.017025:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879393.336967:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879393.336972:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879393.336974:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879393.336978:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879393.336981:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879393.336984:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879393.336986:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879393.336988:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879393.336990:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879394.041015:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879394.041025:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879394.041033:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879394.041038:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879394.041041:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879394.041046:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879394.041050:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879394.041054:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879394.041058:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879394.041062:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879394.041129:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879394.041134:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879394.041139:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879394.361007:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879394.361017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879394.361020:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879394.361027:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879394.361031:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879394.361036:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879394.361039:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879394.361042:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879394.361046:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879395.065013:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879395.065020:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879395.065050:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879395.065052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879395.065095:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879395.065099:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879395.065106:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879395.321113:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879395.321130:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879395.321135:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879395.321141:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879395.321148:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879395.321157:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10334 00000100:00080000:3.0:1788879395.321200:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879395.321205:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879395.321207:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879395.321217:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879395.321221:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879395.321225:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879395.321229:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879395.321231:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879395.321233:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879395.321239:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879395.321243:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879395.321246:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879395.321251:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879395.321255:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879395.321258:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879395.321261:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879395.321274:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879395.321277:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879395.321281:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879395.321284:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879395.321290:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879395.321304:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879395.321317:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290158976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879395.321331:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879395.321337:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879395.321359:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879395.321363:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879395.321368:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879395.321370:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879395.321373:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879395.321377:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879395.321380:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10334 00000100:00080000:3.0:1788879395.321384:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879395.321387:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879395.321389:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879395.321392:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879395.321395:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879395.321398:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879395.321401:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879395.321404:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879395.321406:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879395.321412:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879395.321416:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879395.321418:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879395.321425:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879395.321429:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788879395.321430:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879395.321437:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879395.321438:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00100000:3.0:1788879395.321445:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879395.321446:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879395.321453:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:3.0:1788879395.321455:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.321459:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879395.321461:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879395.321471:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879395.321472:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879395.321479:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879395.321480:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879395.321483:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879395.321489:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879395.321490:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879395.321498:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879395.321505:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290158976:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879395.321509:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.321519:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879395.321527:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1788879395.321528:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290159104/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879395.321533:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879395.321543:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879395.321552:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879395.321553:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879395.321558:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879395.321566:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879395.321569:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879395.321574:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879395.321576:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879395.321578:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788879395.321579:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879395.321586:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:3.0:1788879395.321589:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879395.321595:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879395.321597:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10334 00000100:00080000:3.0:1788879395.321605:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879395.321608:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879395.321610:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879395.321616:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879395.321618:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879395.321620:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879395.321624:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879395.321626:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879395.321628:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879395.321634:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879395.321637:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879395.321641:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879395.321645:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:1.0:1788879395.321653:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879395.321654:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879395.321661:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879395.321665:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879395.321669:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879395.321673:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.321674:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879395.321684:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788879395.321687:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879395.321690:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879395.321692:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.321696:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879395.321698:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879395.321703:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879395.321708:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879395.321709:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.321721:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305007145 00000100:00000001:3.0:1788879395.321724:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879395.321736:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290159232/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879395.321739:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879395.321754:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879395.321757:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879395.321760:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879395.321763:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879395.321767:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305007145 00000100:00080000:3.0:1788879395.321768:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879395.321776:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879395.321781:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788879395.321781:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879395.321792:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879395.321793:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879395.321802:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879395.321806:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10334 00000100:00000001:1.0:1788879395.321809:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788879395.321818:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879395.321825:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1788879395.321828:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879395/real 1788879395] req@ffff94e50449f480 x1875765290158976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879395.321830:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879395.321843:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879395.321848:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879395.321853:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879395.321859:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000400:00000001:1.0:1788879395.321860:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788879395.321863:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000400:00000001:1.0:1788879395.321866:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879395.321871:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879395.321875:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:3.0:1788879395.321882:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879395.321884:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879395.321890:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879395.321892:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.321898:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788879395.321899:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879395.321951:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879395.321957:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879395.321958:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879395.321970:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879395.321976:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879395.321978:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290158976/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1788879395.321982:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879395.321990:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879395.322001:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879395.322007:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879395.322008:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322015:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879395.322018:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879395.322023:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879395.322026:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879395.322028:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879395.322034:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879395.322037:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322038:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879395.322043:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879395.322051:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788879395.322057:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879395.322058:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879395.322065:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879395.322074:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290158976:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1788879395.322082:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290159360/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879395.322084:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322091:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879395.322095:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879395.322100:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879395.322107:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879395.322113:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879395.322118:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788879395.322122:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879395.322125:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1788879395.322130:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879395.322132:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879395.322140:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879395.322143:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322145:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322153:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322162:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879395.322193:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322198:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879395.322203:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879395.322207:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879395.322211:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879395.322214:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879395.322219:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879395.322223:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879395.322228:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879395.322236:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290159104:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879395.322244:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879395.322248:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879395.322251:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879395.322255:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879395.322261:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322274:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879395.322277:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:1.0:1788879395.322285:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879395.322311:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879395.322318:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879395.322322:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879395.322324:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322326:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879395.322330:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322336:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305006604 00000100:00000001:1.0:1788879395.322340:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305007145 00000100:00000001:1.0:1788879395.322343:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305006604 00000100:00000001:1.0:1788879395.322346:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305007145 00000100:00000001:1.0:1788879395.322348:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879395.322351:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322353:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879395.322359:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879395/real 1788879395] req@ffff94e50449c700 x1875765290159104/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879395.322370:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879395.322371:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879395.322375:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879395.322377:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879395.322379:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322381:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879395.322383:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879395.322388:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290159104/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879395.322398:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322400:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879395.322403:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322405:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879395.322407:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322409:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879395.322422:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879395.322425:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879395.322429:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879395.322432:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879395.322436:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290159104:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879395.322440:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879395.322442:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879395.322444:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879395.322447:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879395.322450:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879395.322452:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879395.322456:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290159232:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879395.322460:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879395.322463:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879395.322465:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879395.322467:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879395.322470:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322476:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879395.322478:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:1.0:1788879395.322483:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879395.322496:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879395.322500:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879395.322503:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879395.322506:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322508:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879395.322511:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322515:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305006602 00000100:00000001:1.0:1788879395.322518:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305007145 00000100:00000001:1.0:1788879395.322521:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305006602 00000100:00000001:1.0:1788879395.322524:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305007145 00000100:00000001:1.0:1788879395.322527:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879395.322529:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322531:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879395.322536:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879395/real 1788879395] req@ffff94e50449e300 x1875765290159232/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879395.322545:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879395.322546:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879395.322549:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879395.322550:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879395.322552:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322554:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879395.322556:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879395.322560:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290159232/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879395.322569:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322572:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879395.322575:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322576:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879395.322578:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322580:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879395.322592:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879395.322595:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879395.322598:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879395.322601:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879395.322605:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290159232:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879395.322610:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879395.322611:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879395.322613:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879395.322616:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879395.322618:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879395.322620:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879395.322623:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290159360:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879395.322626:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879395.322628:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879395.322630:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879395.322632:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879395.322635:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322640:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879395.322641:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:1.0:1788879395.322645:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879395.322656:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879395.322660:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879395.322663:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879395.322665:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322667:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879395.322670:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322674:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305006602 00000100:00000001:1.0:1788879395.322677:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305007145 00000100:00000001:1.0:1788879395.322680:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305006602 00000100:00000001:1.0:1788879395.322682:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305007145 00000100:00000001:1.0:1788879395.322685:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879395.322687:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322689:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879395.322693:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879395/real 1788879395] req@ffff94e50449c000 x1875765290159360/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879395.322711:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879395.322712:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879395.322715:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879395.322717:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879395.322719:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322721:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879395.322722:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879395.322727:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290159360/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879395.322734:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322737:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879395.322739:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322741:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879395.322743:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322745:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879395.322756:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879395.322759:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879395.322762:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879395.322765:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879395.322769:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290159360:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879395.322774:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322776:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879395.322777:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879395.322779:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879395.322782:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879395.322784:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879395.322785:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879395.322788:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879395.322790:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879395.322793:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879395.322794:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322797:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879395.322799:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879395.322800:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879395.322803:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879395.322805:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879395.322806:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879395.322808:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879395.322809:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879395.322812:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879395.322813:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322815:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879395.322816:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879395.322818:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879395.322819:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879395.322821:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879395.322822:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879395.322823:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879395.322825:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879395.322826:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879395.322828:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322830:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.322834:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879395.322836:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.322838:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879395.322840:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879395.385049:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879395.385057:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879395.385061:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879395.385069:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879395.385074:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879395.385080:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.385085:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879395.385087:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879395.385091:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879396.089048:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879396.089074:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879396.089078:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879396.089083:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879396.089089:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879396.089092:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879396.089096:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879396.089099:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879396.089105:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879396.089107:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879396.089136:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879396.089140:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879396.089144:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879396.409004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879396.409012:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879396.409014:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879396.409021:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879396.409026:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879396.409030:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879396.409034:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879396.409038:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879396.409058:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879397.113079:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879397.113089:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879397.113096:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879397.113097:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879397.113224:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879397.113228:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879397.113231:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879397.433050:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879397.433061:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879397.433064:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879397.433071:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879397.433077:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879397.433082:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879397.433109:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879397.433114:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879397.433119:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879398.137028:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879398.137049:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879398.137055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879398.137057:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879398.137185:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879398.137192:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879398.137199:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879398.457015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879398.457027:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879398.457030:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879398.457037:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879398.457043:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879398.457048:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879398.457052:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879398.457056:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879398.457060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879399.161041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879399.161050:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879399.161057:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879399.161058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879399.161087:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879399.161090:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879399.161093:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879399.481022:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879399.481033:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879399.481037:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879399.481044:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879399.481050:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879399.481056:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879399.481061:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879399.481064:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879399.481068:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879400.185046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879400.185055:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879400.185063:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879400.185065:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879400.185113:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879400.185116:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879400.185121:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879400.441107:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879400.441118:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879400.441121:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879400.441125:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879400.441151:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879400.441175:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10339 00000100:00080000:2.0:1788879400.441185:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879400.441187:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879400.441188:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879400.441195:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879400.441197:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879400.441201:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879400.441203:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879400.441204:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879400.441205:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879400.441209:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879400.441210:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879400.441213:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879400.441215:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879400.441218:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441237:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879400.441240:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879400.441253:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879400.441255:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441258:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879400.441261:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879400.441266:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441278:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879400.441288:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290159488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879400.441300:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879400.441305:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441326:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879400.441330:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879400.441334:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879400.441335:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879400.441355:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879400.441357:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879400.441360:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10339 00000100:00080000:2.0:1788879400.441362:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879400.441364:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879400.441366:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879400.441368:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879400.441370:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879400.441372:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879400.441374:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879400.441376:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879400.441377:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879400.441379:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879400.441381:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879400.441384:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879400.441389:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879400.441392:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441393:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879400.441397:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879400.441400:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441402:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879400.441407:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879400.441408:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441411:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879400.441413:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879400.441415:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441420:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879400.441425:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290159616/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879400.441433:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879400.441434:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441436:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879400.441439:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879400.441442:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879400.441445:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879400.441447:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788879400.441448:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879400.441452:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879400.441460:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10339 00000100:00080000:2.0:1788879400.441463:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879400.441466:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879400.441467:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879400.441478:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879400.441480:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788879400.441482:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879400.441487:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879400.441490:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1788879400.441491:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879400.441502:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879400.441503:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879400.441513:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879400.441517:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879400.441521:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788879400.441523:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879400.441525:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879400.441532:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879400.441540:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879400.441541:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879400.441553:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441559:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879400.441561:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788879400.441562:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879400.441573:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441576:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879400.441579:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879400.441580:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441585:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879400.441587:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1788879400.441589:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290159488:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879400.441589:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441602:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.441618:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:2.0:1788879400.441619:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290159744/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879400.441645:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788879400.441652:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879400.441655:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441656:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441671:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879400.441673:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1788879400.441678:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879400.441683:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879400.441685:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879400.441686:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879400.441690:0:178195:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:1.0:1788879400.441693:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879400.441693:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10339 00000100:00080000:2.0:1788879400.441701:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879400.441720:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879400.441723:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879400.441743:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879400.441743:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879400.441745:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879400.441746:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879400.441748:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879400.441748:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879400.441751:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879400.441752:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879400.441753:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879400.441755:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879400.441756:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441757:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879400.441759:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879400.441760:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441762:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879400.441765:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879400.441768:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788879400.441769:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441777:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879400.441778:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 02000000:00000001:2.0:1788879400.441780:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879400.441782:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.441796:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879400.441796:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879400.441806:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290159872/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879400.441812:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879400.441815:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879400.441816:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879400.441881:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879400.441967:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879400.441977:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879400.441982:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.441987:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879400.441997:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.442007:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305012266 00000100:00000001:1.0:1788879400.442026:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788879400.442039:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305012266 00000100:00000001:1.0:1788879400.442046:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879400.442053:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879400.442059:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.442065:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879400.442078:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879400/real 1788879400] req@ffff94e5045e6a00 x1875765290159488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879400.442102:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879400.442106:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879400.442114:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879400.442119:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879400.442123:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.442127:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879400.442163:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879400.442175:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290159488/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879400.442194:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.442201:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879400.442206:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.442210:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879400.442213:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.442218:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879400.442223:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879400.442229:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879400.442239:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879400.442244:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879400.442252:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290159488:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879400.442261:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879400.442264:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879400.442268:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879400.442272:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879400.442276:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879400.442281:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879400.442287:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290159616:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879400.442295:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879400.442299:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879400.442302:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879400.442306:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879400.442313:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.442326:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879400.442329:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:1.0:1788879400.442337:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879400.442366:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879400.442376:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879400.442381:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879400.442384:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.442387:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879400.442394:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.442402:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305011724 00000100:00000001:1.0:1788879400.442408:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305012266 00000100:00000001:1.0:1788879400.442412:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305011724 00000100:00000001:1.0:1788879400.442416:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305012266 00000100:00000001:1.0:1788879400.442420:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879400.442425:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.442429:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879400.442438:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879400/real 1788879400] req@ffff94e5045e5880 x1875765290159616/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879400.442456:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879400.442459:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879400.442465:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879400.442468:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879400.442471:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.442475:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879400.442477:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879400.442486:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290159616/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879400.442500:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.442505:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879400.442510:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.442513:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879400.442515:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.442519:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879400.442545:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879400.442551:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879400.442559:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879400.442564:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879400.442571:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290159616:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879400.442580:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.442584:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879400.442587:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879400.442593:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879400.442601:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879400.442605:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879400.442609:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879400.442617:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879400.442620:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879400.442626:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879400.442629:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.442633:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879400.442636:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879400.442638:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879400.442641:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879400.442644:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879400.442646:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879400.442650:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879400.442652:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879400.442655:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879400.442658:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.442663:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.442676:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879400.442680:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.442686:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879400.442689:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879400.442692:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879400.442695:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879400.442698:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879400.442703:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879400.442707:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879400.442711:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879400.442719:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290159744:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879400.442765:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879400.442770:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879400.442773:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879400.442777:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879400.442783:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.442797:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879400.442800:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 00000100:00000001:1.0:1788879400.442807:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879400.442833:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879400.442843:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879400.442848:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879400.442850:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.442854:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879400.442859:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.442867:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305011722 00000100:00000001:1.0:1788879400.442873:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305012266 00000100:00000001:1.0:1788879400.442877:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305011722 00000100:00000001:1.0:1788879400.442882:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305012266 00000100:00000001:1.0:1788879400.442887:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879400.442891:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.442895:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879400.442950:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879400/real 1788879400] req@ffff94e5045e5180 x1875765290159744/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879400.442970:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879400.442973:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879400.442979:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879400.442983:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879400.442986:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.442990:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879400.442992:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879400.443001:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290159744/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879400.443026:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.443031:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879400.443036:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.443039:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879400.443042:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.443048:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879400.443072:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879400.443077:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879400.443085:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879400.443089:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879400.443096:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290159744:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879400.443105:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879400.443108:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879400.443111:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879400.443116:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879400.443121:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879400.443125:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879400.443178:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290159872:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879400.443187:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879400.443192:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879400.443195:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879400.443199:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879400.443205:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.443217:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879400.443220:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:1.0:1788879400.443227:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879400.443256:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879400.443265:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879400.443270:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879400.443273:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.443276:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879400.443282:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.443290:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305011723 00000100:00000001:1.0:1788879400.443295:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305012266 00000100:00000001:1.0:1788879400.443300:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305011723 00000100:00000001:1.0:1788879400.443305:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305012266 00000100:00000001:1.0:1788879400.443312:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879400.443317:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.443320:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879400.443329:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879400/real 1788879400] req@ffff94e5045e4a80 x1875765290159872/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879400.443356:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879400.443359:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879400.443364:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879400.443368:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879400.443370:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.443374:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879400.443377:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879400.443386:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290159872/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879400.443401:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.443405:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879400.443410:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.443413:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879400.443416:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.443419:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879400.443443:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879400.443449:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879400.443458:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879400.443463:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879400.443470:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290159872:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879400.443478:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.443482:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879400.443485:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879400.443488:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879400.443494:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879400.443498:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879400.443501:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879400.443506:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879400.443508:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879400.443512:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879400.443526:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.443530:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879400.443532:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879400.443535:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879400.443539:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879400.443541:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879400.443544:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879400.443547:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879400.443550:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879400.443553:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879400.443556:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.443559:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.443567:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879400.443570:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.443574:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879400.443578:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879400.505004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879400.505014:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879400.505017:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879400.505024:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879400.505029:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879400.505034:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.505038:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879400.505042:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879400.505045:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879401.209026:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879401.209034:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879401.209041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879401.209043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879401.209085:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879401.209088:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879401.209090:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879401.529083:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879401.529094:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879401.529098:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879401.529105:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879401.529111:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879401.529116:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879401.529120:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879401.529159:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879401.529181:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879402.233035:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879402.233043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879402.233068:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879402.233071:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879402.233189:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879402.233200:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879402.233211:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879402.553166:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879402.553179:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879402.553182:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879402.553189:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879402.553195:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879402.553201:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879402.553204:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879402.553208:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879402.553212:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879403.257051:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879403.257056:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879403.257061:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879403.257062:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879403.257084:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879403.257086:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879403.257089:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879403.577078:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879403.577087:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879403.577090:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879403.577096:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879403.577100:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879403.577104:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879403.577123:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879403.577126:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879403.577160:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879404.281042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879404.281052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879404.281058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879404.281062:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879404.281066:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879404.281070:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879404.281074:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879404.281077:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879404.281079:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879404.281088:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879404.281149:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879404.281153:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879404.281156:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879404.601028:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879404.601037:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879404.601039:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879404.601044:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879404.601047:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879404.601051:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879404.601054:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879404.601056:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879404.601059:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879405.305012:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879405.305020:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879405.305025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879405.305026:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879405.305049:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879405.305051:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879405.305065:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879405.561019:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879405.561033:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879405.561036:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879405.561042:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879405.561048:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879405.561058:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10344 00000100:00080000:2.0:1788879405.561072:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879405.561076:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879405.561079:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879405.561091:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879405.561116:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879405.561123:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879405.561127:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879405.561132:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879405.561134:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879405.561142:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879405.561147:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879405.561153:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879405.561159:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879405.561165:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561169:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879405.561174:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879405.561192:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879405.561194:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561198:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879405.561201:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879405.561209:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561225:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879405.561238:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290160000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879405.561252:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879405.561258:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561282:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879405.561289:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879405.561296:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879405.561299:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879405.561301:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879405.561307:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879405.561312:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10344 00000100:00080000:2.0:1788879405.561321:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879405.561326:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879405.561330:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879405.561337:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879405.561338:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788879405.561341:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879405.561349:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879405.561350:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788879405.561357:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879405.561359:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879405.561364:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879405.561368:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879405.561369:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879405.561377:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879405.561378:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879405.561387:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879405.561389:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879405.561391:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879405.561399:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879405.561400:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879405.561425:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879405.561429:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561430:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879405.561440:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879405.561448:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290160000:192.168.203.129@tcp:250:kworker.0 00000008:00000001:2.0:1788879405.561449:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561454:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879405.561465:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879405.561470:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.561477:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788879405.561477:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879405.561483:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879405.561489:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879405.561490:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879405.561498:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561498:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879405.561509:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1788879405.561516:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290160128/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879405.561523:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561544:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879405.561548:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561553:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879405.561556:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879405.561562:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788879405.561562:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879405.561570:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879405.561571:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00080000:2.0:1788879405.561574:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879405.561583:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879405.561586:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788879405.561589:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10344 00000100:00080000:2.0:1788879405.561599:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879405.561602:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879405.561603:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879405.561608:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879405.561610:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879405.561613:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879405.561617:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879405.561620:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879405.561622:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879405.561627:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879405.561630:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879405.561632:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879405.561636:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879405.561639:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561642:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879405.561645:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879405.561648:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561650:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879405.561657:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879405.561659:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561666:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000100:1.0:1788879405.561670:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:2.0:1788879405.561672:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879405.561679:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561693:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.561698:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00100000:2.0:1788879405.561709:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290160256/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879405.561712:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879405.561719:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.561730:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879405.561742:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879405.561744:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561750:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.561756:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305017385 00000100:00000001:2.0:1788879405.561761:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879405.561768:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879405.561773:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879405.561776:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879405.561779:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1788879405.561780:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879405.561793:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879405.561798:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10344 00000100:00080000:2.0:1788879405.561804:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879405.561806:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879405.561808:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879405.561814:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305017385 00000020:00000001:2.0:1788879405.561815:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879405.561820:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879405.561823:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879405.561829:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879405.561832:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879405.561833:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879405.561838:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879405.561848:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879405.561850:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879405.561853:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879405.561862:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879405.561864:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879405.561868:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879405.561875:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.561876:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879405.561879:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879405.561886:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879405.561891:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561897:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879405.561900:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879405/real 1788879405] req@ffff94e5045e7100 x1875765290160000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879405.561903:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879405.561906:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561909:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879405.561910:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879405.561913:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879405.561967:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879405.561992:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:2.0:1788879405.561999:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4380 x1875765290160384/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1788879405.562000:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879405.562014:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879405.562015:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879405.562022:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.562026:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879405.562027:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.562035:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562043:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879405.562047:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879405.562059:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290160000/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879405.562074:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562081:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879405.562085:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562087:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879405.562089:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562092:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879405.562117:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879405.562121:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879405.562127:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879405.562135:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879405.562140:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290160000:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879405.562147:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562150:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879405.562152:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879405.562155:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879405.562161:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879405.562164:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879405.562166:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879405.562172:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879405.562174:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879405.562178:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879405.562179:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562182:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562191:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879405.562194:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.562196:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879405.562198:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879405.562200:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879405.562202:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879405.562204:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879405.562206:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879405.562208:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879405.562211:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879405.562215:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290160128:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879405.562219:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879405.562221:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879405.562224:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879405.562226:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879405.562230:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.562239:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879405.562240:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:1.0:1788879405.562246:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879405.562264:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879405.562270:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879405.562273:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879405.562274:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.562276:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879405.562280:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.562284:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305016844 00000100:00000001:1.0:1788879405.562288:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305017385 00000100:00000001:1.0:1788879405.562291:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305016844 00000100:00000001:1.0:1788879405.562293:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305017385 00000100:00000001:1.0:1788879405.562296:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879405.562298:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.562300:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879405.562305:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879405/real 1788879405] req@ffff94e5045e5c00 x1875765290160128/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879405.562314:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879405.562315:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879405.562318:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879405.562319:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879405.562321:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562323:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879405.562325:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879405.562329:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290160128/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879405.562336:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562339:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879405.562342:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562344:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879405.562346:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562348:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879405.562366:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879405.562370:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879405.562374:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879405.562376:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879405.562380:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290160128:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879405.562384:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879405.562385:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879405.562387:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879405.562389:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879405.562391:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879405.562393:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879405.562413:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290160256:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879405.562417:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879405.562420:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879405.562422:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879405.562424:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879405.562427:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.562434:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879405.562436:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:1.0:1788879405.562441:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879405.562455:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879405.562459:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879405.562462:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879405.562464:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.562466:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879405.562470:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.562474:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305016842 00000100:00000001:1.0:1788879405.562477:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305017385 00000100:00000001:1.0:1788879405.562480:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305016842 00000100:00000001:1.0:1788879405.562483:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305017385 00000100:00000001:1.0:1788879405.562486:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879405.562488:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.562491:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879405.562495:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879405/real 1788879405] req@ffff94e5045e5500 x1875765290160256/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879405.562505:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879405.562506:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879405.562508:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879405.562510:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879405.562512:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562514:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879405.562515:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879405.562520:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290160256/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879405.562529:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562531:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879405.562534:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562536:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879405.562538:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562540:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879405.562557:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879405.562559:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879405.562563:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879405.562566:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879405.562569:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290160256:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879405.562574:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879405.562576:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879405.562578:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879405.562581:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879405.562583:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879405.562586:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879405.562589:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290160384:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879405.562593:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879405.562594:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879405.562596:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879405.562598:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879405.562601:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.562606:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879405.562607:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074549120 : -117763635002496 : ffff94e5045e4380) 00000100:00000001:1.0:1788879405.562611:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879405.562622:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879405.562626:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879405.562628:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879405.562630:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.562632:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879405.562635:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.562639:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305016842 00000100:00000001:1.0:1788879405.562642:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305017385 00000100:00000001:1.0:1788879405.562645:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305016842 00000100:00000001:1.0:1788879405.562647:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305017385 00000100:00000001:1.0:1788879405.562650:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879405.562652:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.562654:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879405.562659:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879405/real 1788879405] req@ffff94e5045e4380 x1875765290160384/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879405.562667:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879405.562669:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879405.562671:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879405.562673:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879405.562674:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562677:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879405.562679:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879405.562684:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4380 x1875765290160384/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879405.562693:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562695:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879405.562698:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562700:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879405.562701:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562703:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879405.562719:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879405.562722:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879405.562725:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879405.562727:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879405.562731:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290160384:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879405.562735:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562737:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879405.562739:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879405.562741:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879405.562743:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879405.562745:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879405.562747:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879405.562750:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879405.562751:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879405.562754:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879405.562756:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562758:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879405.562759:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879405.562761:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879405.562763:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879405.562764:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879405.562766:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879405.562768:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879405.562769:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879405.562771:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879405.562774:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562776:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879405.562777:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879405.562779:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879405.562782:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879405.562783:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879405.562785:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879405.562787:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879405.562788:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879405.562790:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879405.562791:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562793:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.562797:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879405.562799:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.562801:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879405.562803:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879405.625079:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879405.625117:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879405.625120:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879405.625128:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879405.625144:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879405.625151:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.625156:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879405.625161:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879405.625166:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879406.328988:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879406.328995:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879406.328998:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879406.329001:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879406.329005:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879406.329007:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879406.329009:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879406.329010:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879406.329013:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879406.329015:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879406.329083:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879406.329087:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879406.329106:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879406.649019:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879406.649029:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879406.649032:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879406.649038:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879406.649043:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879406.649047:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879406.649050:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879406.649053:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879406.649056:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879407.353062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879407.353070:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879407.353075:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879407.353076:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879407.353135:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879407.353137:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879407.353140:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879407.673022:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879407.673029:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879407.673032:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879407.673038:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879407.673042:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879407.673046:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879407.673049:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879407.673052:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879407.673054:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879408.377007:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879408.377014:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879408.377020:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879408.377021:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879408.377045:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879408.377048:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879408.377053:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879408.697045:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879408.697057:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879408.697061:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879408.697071:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879408.697114:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879408.697121:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879408.697126:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879408.697131:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879408.697135:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879409.400985:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879409.400991:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879409.401005:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879409.401006:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879409.401028:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879409.401031:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879409.401033:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879409.721036:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879409.721043:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879409.721044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879409.721049:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879409.721052:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879409.721055:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879409.721058:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879409.721060:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879409.721062:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879410.424983:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879410.424988:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879410.424992:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879410.424993:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879410.425041:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879410.425044:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879410.425047:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879410.681111:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879410.681123:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879410.681126:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879410.681131:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879410.681136:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879410.681144:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10349 00000100:00080000:3.0:1788879410.681156:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879410.681160:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879410.681165:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879410.681177:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879410.681181:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879410.681187:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879410.681192:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879410.681195:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879410.681198:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879410.681205:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879410.681209:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879410.681212:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879410.681216:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879410.681220:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879410.681223:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879410.681226:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879410.681238:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879410.681241:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879410.681245:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879410.681248:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879410.681252:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879410.681266:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879410.681298:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290160512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879410.681311:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879410.681316:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879410.681365:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879410.681372:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879410.681380:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879410.681384:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879410.681388:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879410.681406:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879410.681418:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10349 00000100:00080000:3.0:1788879410.681431:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879410.681437:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879410.681443:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879410.681444:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879410.681458:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788879410.681462:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879410.681464:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879410.681478:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1788879410.681478:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879410.681487:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879410.681492:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:2.0:1788879410.681499:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879410.681510:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788879410.681510:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:2.0:1788879410.681525:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879410.681540:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290160512:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879410.681548:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879410.681556:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879410.681559:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879410.681563:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879410.681571:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.681600:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879410.681603:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:2.0:1788879410.681610:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879410.681637:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879410.681646:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879410.681659:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879410.681666:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879410.681670:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879410.681682:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:2.0:1788879410.681693:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879410.681720:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879410.681730:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879410.681735:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.681738:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879410.681748:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.681755:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305022505 00000100:00000001:2.0:1788879410.681770:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788879410.681781:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305022505 00000100:00000001:2.0:1788879410.681786:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879410.681791:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879410.681795:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.681799:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879410.681811:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879410/real 1788879410] req@ffff94e50449f100 x1875765290160512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879410.681829:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879410.681832:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879410.681839:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879410.681842:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879410.681844:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.681860:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879410.681863:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879410.681869:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290160512/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879410.681880:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.681885:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879410.681890:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.681893:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879410.681894:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.681898:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879410.681901:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879410.681974:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879410.681980:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879410.681984:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879410.681989:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290160512:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879410.681997:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.682002:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879410.682004:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879410.682008:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879410.682026:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879410.682031:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879410.682034:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879410.682041:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879410.682048:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:3.0:1788879410.682048:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879410.682055:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879410.682082:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000008:00100000:3.0:1788879410.682083:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879410.682092:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1788879410.682093:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.682114:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879410.682127:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.682129:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879410.682134:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879410.682140:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879410.682141:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788879410.682148:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.682150:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879410.682162:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879410.682168:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879410.682183:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879410.682194:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879410.682207:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290160640/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879410.682219:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879410.682224:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879410.682235:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879410.682240:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879410.682245:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879410.682249:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879410.682253:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1788879410.682253:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879410.682261:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879410.682265:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879410.682266:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879410.682273:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879410.682278:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879410.682280:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10349 02000000:00000001:2.0:1788879410.682295:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879410.682301:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879410.682308:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879410.682317:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290160640:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879410.682322:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879410.682327:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879410.682330:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879410.682334:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879410.682338:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.682352:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879410.682354:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:2.0:1788879410.682361:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879410.682381:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879410.682389:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879410.682394:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:2.0:1788879410.682401:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:3.0:1788879410.682408:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879410.682412:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:3.0:1788879410.682413:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879410.682419:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879410.682422:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1788879410.682430:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879410.682431:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.682453:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879410.682453:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879410.682472:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879410.682474:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.682487:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305021964 00000100:00080000:3.0:1788879410.682489:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879410.682501:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305022505 00000100:00000001:3.0:1788879410.682502:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.682520:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305021964 02000000:00000001:3.0:1788879410.682521:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879410.682532:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305022505 02000000:00000001:3.0:1788879410.682535:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879410.682537:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879410.682544:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.682545:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879410.682552:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879410.682554:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000008:00100000:3.0:1788879410.682572:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879410.682579:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879410/real 1788879410] req@ffff94e50449ca80 x1875765290160640/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1788879410.682581:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879410.682592:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879410.682615:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000400:00000001:2.0:1788879410.682617:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1788879410.682629:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879410.682630:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879410.682638:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879410.682642:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879410.682644:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:3.0:1788879410.682646:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879410.682668:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.682672:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879410.682681:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879410.682682:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290160768/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879410.682696:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879410.682710:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879410.682711:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879410.682720:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879410.682730:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879410.682735:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290160640/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879410.682736:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879410.682763:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879410.682770:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879410.682778:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879410.682798:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879410.682799:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.682815:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879410.682815:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10349 00000100:00000001:2.0:1788879410.682834:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788879410.682837:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879410.682840:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879410.682844:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879410.682844:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879410.682854:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879410.682857:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000020:00000001:3.0:1788879410.682874:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879410.682879:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879410.682884:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879410.682891:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879410.682897:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879410.682899:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879410.682902:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879410.682906:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879410.682983:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00080000:2.0:1788879410.682984:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879410.683003:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788879410.683005:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879410.683032:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290160640:192.168.203.129@tcp:38:kworker.0 02000000:00000001:3.0:1788879410.683035:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879410.683051:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879410.683051:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879410.683097:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.683100:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879410.683103:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879410.683105:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000008:00100000:3.0:1788879410.683108:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788879410.683110:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000008:00000001:3.0:1788879410.683118:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879410.683119:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1788879410.683124:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879410.683125:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879410.683131:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879410.683136:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879410.683137:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879410.683145:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:3.0:1788879410.683146:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.683150:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879410.683153:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879410.683154:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879410.683160:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879410.683161:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879410.683171:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.683173:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879410.683177:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879410.683181:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.683182:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879410.683187:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879410.683190:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879410.683194:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879410.683198:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879410.683198:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290160896/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788879410.683208:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879410.683215:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879410.683230:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879410.683237:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1788879410.683243:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290160768:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879410.683247:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.683257:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879410.683260:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879410.683261:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879410.683265:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879410.683269:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879410.683272:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.683282:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879410.683284:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:2.0:1788879410.683289:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879410.683307:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879410.683312:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879410.683314:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879410.683316:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.683317:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879410.683320:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.683323:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305021963 00000100:00000001:2.0:1788879410.683325:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305022505 00000100:00000001:2.0:1788879410.683328:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305021963 00000100:00000001:2.0:1788879410.683330:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305022505 00000100:00000001:2.0:1788879410.683333:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879410.683334:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.683336:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879410.683340:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879410/real 1788879410] req@ffff94e50449f100 x1875765290160768/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879410.683348:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879410.683349:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879410.683352:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879410.683353:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879410.683355:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.683357:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879410.683358:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879410.683361:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290160768/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879410.683376:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.683378:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879410.683381:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.683382:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879410.683384:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.683385:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879410.683402:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879410.683404:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879410.683407:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879410.683409:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879410.683412:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290160768:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879410.683416:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.683418:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879410.683420:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879410.683421:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879410.683423:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879410.683425:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879410.683426:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879410.683428:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879410.683430:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879410.683432:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879410.683433:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.683435:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.683438:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879410.683439:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.683441:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879410.683443:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879410.683445:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879410.683446:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879410.683448:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879410.683450:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879410.683451:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879410.683453:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879410.683455:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290160896:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879410.683459:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879410.683460:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879410.683462:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879410.683463:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879410.683465:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.683469:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879410.683471:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:2.0:1788879410.683474:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879410.683482:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879410.683486:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879410.683487:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879410.683489:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.683490:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879410.683492:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.683496:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305021963 00000100:00000001:2.0:1788879410.683498:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305022505 00000100:00000001:2.0:1788879410.683500:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305021963 00000100:00000001:2.0:1788879410.683503:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305022505 00000100:00000001:2.0:1788879410.683505:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879410.683507:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.683509:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879410.683513:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879410/real 1788879410] req@ffff94e50449c000 x1875765290160896/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879410.683519:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879410.683520:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879410.683521:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879410.683523:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879410.683524:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.683525:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879410.683526:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879410.683529:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290160896/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879410.683536:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.683537:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879410.683539:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.683540:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879410.683541:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.683543:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879410.683559:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879410.683561:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879410.683563:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879410.683565:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879410.683567:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290160896:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879410.683570:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.683572:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879410.683573:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879410.683574:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879410.683577:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879410.683578:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879410.683579:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879410.683581:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879410.683582:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879410.683584:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879410.683585:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.683587:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879410.683589:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879410.683591:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879410.683592:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879410.683594:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879410.745049:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879410.745078:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879410.745082:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879410.745090:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879410.745113:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879410.745119:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879410.745123:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879410.745128:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879410.745132:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879411.449071:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879411.449082:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879411.449090:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879411.449093:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879411.449132:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879411.449137:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879411.449142:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879411.769028:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879411.769038:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879411.769042:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879411.769077:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879411.769093:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879411.769111:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879411.769115:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879411.769118:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879411.769122:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879412.473024:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879412.473032:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879412.473038:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879412.473040:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879412.473109:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879412.473114:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879412.473120:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879412.793071:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879412.793080:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879412.793084:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879412.793091:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879412.793096:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879412.793101:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879412.793104:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879412.793108:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879412.793112:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879413.496992:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879413.496997:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879413.497001:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879413.497002:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879413.497019:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879413.497021:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879413.497023:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879413.817063:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879413.817073:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879413.817076:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879413.817083:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879413.817087:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879413.817092:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879413.817095:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879413.817099:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879413.817102:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879414.521020:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879414.521029:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879414.521055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879414.521060:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879414.521063:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879414.521069:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879414.521073:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879414.521076:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879414.521097:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879414.521101:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879414.521197:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879414.521202:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879414.521207:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879414.841027:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879414.841086:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879414.841088:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879414.841096:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879414.841102:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879414.841107:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879414.841111:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879414.841115:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879414.841118:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879415.559593:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879415.561154:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879415.561166:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879415.561168:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879415.565957:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879415.565961:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879415.565964:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879415.801227:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879415.801245:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879415.801252:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879415.801258:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879415.801284:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879415.801326:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10354 00000100:00080000:2.0:1788879415.801344:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879415.801350:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879415.801354:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879415.801366:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879415.801372:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879415.801377:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879415.801382:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879415.801386:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879415.801389:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879415.801399:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879415.801404:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879415.801410:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879415.801417:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879415.801422:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.801427:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879415.801431:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879415.801450:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879415.801453:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.801460:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879415.801464:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879415.801471:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.801490:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879415.801505:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290161024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879415.801524:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879415.801532:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.801560:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879415.801590:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879415.801598:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879415.801600:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879415.801603:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879415.801609:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879415.801615:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10354 00000100:00080000:2.0:1788879415.801623:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879415.801627:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879415.801630:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879415.801636:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879415.801639:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879415.801643:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879415.801649:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879415.801654:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879415.801657:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879415.801669:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879415.801677:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879415.801681:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1788879415.801684:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879415.801686:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879415.801708:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.801717:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879415.801718:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879415.801723:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879415.801727:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879415.801738:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1788879415.801742:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290161024:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879415.801750:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879415.801753:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879415.801756:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879415.801763:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879415.801766:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879415.801768:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879415.801776:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.801803:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879415.801811:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879415.801819:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.801823:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.801831:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788879415.801834:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879415.801836:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 02000000:00000001:2.0:1788879415.801843:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.801845:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879415.801854:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879415.801860:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879415.801864:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.801878:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879415.801893:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5f80 x1875765290161152/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879415.801974:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879415.801980:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.801990:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879415.801997:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879415.802002:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879415.802005:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879415.802010:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305027626 00000100:00000001:2.0:1788879415.802012:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879415.802020:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879415.802068:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879415.802070:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1788879415.802078:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10354 00000100:00000001:1.0:1788879415.802088:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305027626 00000100:00080000:2.0:1788879415.802093:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879415.802103:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879415.802105:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879415.802108:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879415.802118:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1788879415.802121:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879415.802130:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879415.802131:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879415.802139:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879415.802142:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879415.802151:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879415.802154:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879415.802164:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879415.802171:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879415.802173:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879415.802181:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00080000:2.0:1788879415.802187:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879415.802188:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879415.802194:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879415.802201:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879415.802202:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.802203:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879415.802209:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879415.802213:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879415.802215:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879415.802222:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879415.802225:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879415.802227:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879415.802238:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.802242:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879415.802248:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879415.802249:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290161152:192.168.203.129@tcp:38:kworker.0 00000008:00000001:2.0:1788879415.802260:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.802261:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879415.802268:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788879415.802288:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879415.802291:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879415.802298:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879415.802304:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879415.802305:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879415.802311:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.802315:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879415.802318:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788879415.802319:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879415.802323:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074556288 : -117763634995328 : ffff94e5045e5f80) 02000000:00000001:2.0:1788879415.802327:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.802331:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879415.802346:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.802358:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879415.802363:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290161280/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879415.802366:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305027084 00000100:00000001:1.0:1788879415.802375:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305027626 00000100:00000001:1.0:1788879415.802384:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305027084 00000100:00000001:1.0:1788879415.802391:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305027626 00000100:00000001:1.0:1788879415.802398:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879415.802404:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879415.802405:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.802413:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.802417:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.802418:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879415.802431:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879415.802433:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.802434:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788879415.802444:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788879415.802447:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879415.802449:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879415.802454:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879415.802458:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788879415.802464:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879415.802465:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879415.802472:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879415.802474:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1788879415.802478:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788879415.802481:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1788879415.802483:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.802495:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00100000:1.0:1788879415.802496:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290161280:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879415.802504:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1788879415.802510:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10354 02000000:00000001:1.0:1788879415.802511:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879415.802515:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879415.802521:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879415.802525:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879415.802526:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879415.802532:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879415.802534:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879415.802536:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 00000100:00000001:2.0:1788879415.802541:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879415.802544:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1788879415.802555:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879415.802559:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879415.802560:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879415.802565:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879415.802567:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305027082 00000020:00000001:2.0:1788879415.802577:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879415.802578:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305027626 00000100:00000001:1.0:1788879415.802584:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305027082 00000100:00000001:2.0:1788879415.802591:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879415.802592:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305027626 00000100:00000001:2.0:1788879415.802599:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879415.802600:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879415.802609:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.802613:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879415.802616:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879415.802617:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879415.802623:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879415.802626:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.802627:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788879415.802638:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879415.802638:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879415.802643:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879415.802648:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.802651:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879415.802651:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879415.802662:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.802668:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879415.802672:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879415.802678:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.802681:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879415.802689:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879415.802692:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.802696:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879415.802699:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879415.802703:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.802730:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879415.802741:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4380 x1875765290161408/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879415.802755:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879415.802758:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879415.802792:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.803081:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879415.803086:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879415.803088:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879415.803090:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879415.803092:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879415.803097:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879415.803100:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879415.803103:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879415.803110:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290161408:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879415.803117:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879415.803119:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879415.803122:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879415.803124:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879415.803129:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.803140:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879415.803142:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074549120 : -117763635002496 : ffff94e5045e4380) 00000100:00000001:1.0:1788879415.803148:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879415.803169:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.803175:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305027083 00000100:00000001:1.0:1788879415.803179:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305027626 00000100:00000001:1.0:1788879415.803183:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305027083 00000100:00000001:1.0:1788879415.803195:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305027626 00000100:00000001:1.0:1788879415.803199:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879415.803202:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.803205:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.803208:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879415.803214:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879415.803216:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788879415.803219:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879415.803221:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879415.803223:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.803226:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879415.865049:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879415.865058:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879415.865061:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879415.865080:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879415.865085:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879415.865090:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879415.865094:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879415.865098:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879415.865102:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879416.569058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879416.569068:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879416.569072:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879416.569077:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879416.569083:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879416.569086:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879416.569089:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879416.569091:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879416.569097:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879416.569099:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879416.569124:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879416.569128:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879416.569132:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879416.889010:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879416.889045:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879416.889048:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879416.889066:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879416.889071:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879416.889076:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879416.889080:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879416.889083:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879416.889086:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879417.593056:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879417.593067:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879417.593077:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879417.593080:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879417.593111:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879417.593115:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879417.593119:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879417.913077:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879417.913096:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879417.913099:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879417.913107:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879417.913112:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879417.913117:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879417.913121:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879417.913125:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879417.913128:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879418.617040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879418.617048:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879418.617055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879418.617057:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879418.617153:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879418.617159:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879418.617164:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788879418.873194:0:2351:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1788879418.873248:0:2351:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788879418.873267:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879418.873283:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879418.873305:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879418.873308:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879418.873311:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879418.873323:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879418.873330:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879418.873334:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879418.873337:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879418.873339:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879418.873346:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879418.873355:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879418.873355:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879418.873360:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879418.873363:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879418.873364:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879418.873368:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879418.873371:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000800:00000100:0.0:1788879418.873375:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879418.873380:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879418.873383:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879418.873385:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879418.873387:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:1.0:1788879418.873389:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879415/real 1788879418] req@ffff94e5045e6300 x1875765290161280/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879418.873421:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879418.873423:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879418.873431:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879418.873435:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879418.873438:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873441:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879418.873443:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879418.873450:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290161280/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879418.873468:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873472:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879418.873476:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873478:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879418.873479:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873482:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879418.873532:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879418.873538:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879418.873545:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879418.873548:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879418.873553:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290161280:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879418.873559:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879418.873564:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879415/real 1788879418] req@ffff94e5045e5f80 x1875765290161152/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879418.873574:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879418.873576:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879418.873580:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879418.873582:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879418.873584:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873586:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879418.873587:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879418.873593:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5f80 x1875765290161152/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879418.873601:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873603:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879418.873606:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873608:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879418.873609:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873611:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879418.873629:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879418.873633:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879418.873638:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879418.873640:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879418.873645:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290161152:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879418.873650:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879418.873655:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879415/real 1788879418] req@ffff94e5045e7800 x1875765290161024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879418.873664:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879418.873666:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879418.873671:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879418.873673:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879418.873675:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873677:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879418.873678:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879418.873684:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290161024/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879418.873693:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873695:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879418.873698:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873700:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879418.873702:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873703:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879418.873705:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879418.873709:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879418.873713:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879418.873716:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879418.873719:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290161024:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879418.873725:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879418.873728:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879418.873730:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879418.873733:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879418.873741:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879418.873744:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879418.873747:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879418.873751:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879418.873754:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879418.873758:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879418.873760:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873763:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879418.873765:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879418.873767:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879418.873769:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879418.873770:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879418.873772:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879418.873775:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879418.873776:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879418.873778:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879418.873780:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873782:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879418.873784:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879418.873785:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879418.873787:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879418.873797:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879418.873798:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879418.873800:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879418.873802:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879418.873804:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879418.873805:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873808:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879418.873810:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879418.873812:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879418.873813:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879418.873819:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879415/real 1788879418] req@ffff94e5045e4380 x1875765290161408/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879418.873830:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879418.873831:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879418.873835:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879418.873837:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879418.873838:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873841:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879418.873842:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879418.873847:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4380 x1875765290161408/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879418.873855:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873858:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879418.873860:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873862:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879418.873863:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873865:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879418.873882:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879418.873886:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879418.873890:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879418.873893:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879418.873897:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290161408:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879418.873930:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873934:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879418.873936:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879418.873939:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879418.873943:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879418.873945:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879418.873947:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879418.873950:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879418.873952:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879418.873956:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879418.873957:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873960:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.873969:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879418.873972:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879418.873975:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879418.873977:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879418.937019:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879418.937028:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879418.937030:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879418.937036:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879418.937039:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879418.937069:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.937072:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879418.937076:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879418.937079:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879419.641045:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879419.641056:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879419.641067:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879419.641069:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879419.641102:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879419.641107:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879419.641113:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879419.961068:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879419.961080:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879419.961083:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879419.961093:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879419.961098:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879419.961104:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879419.961108:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879419.961114:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879419.961118:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879420.665069:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879420.665078:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879420.665085:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879420.665086:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879420.665116:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879420.665119:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879420.665123:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879420.921092:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879420.921104:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879420.921106:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879420.921110:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879420.921115:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879420.921124:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10359 00000100:00080000:3.0:1788879420.921136:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879420.921139:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879420.921141:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879420.921151:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879420.921153:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879420.921157:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879420.921159:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879420.921161:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879420.921162:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879420.921166:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879420.921168:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879420.921171:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879420.921175:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879420.921178:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879420.921180:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879420.921183:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879420.921193:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879420.921195:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879420.921198:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879420.921200:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879420.921205:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879420.921220:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879420.921230:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290161536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879420.921239:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879420.921244:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879420.921261:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879420.921264:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879420.921267:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879420.921268:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879420.921269:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879420.921271:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879420.921273:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10359 00000100:00080000:3.0:1788879420.921276:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879420.921277:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879420.921278:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879420.921299:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879420.921301:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879420.921303:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879420.921306:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879420.921309:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879420.921311:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879420.921314:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879420.921319:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879420.921319:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879420.921325:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879420.921329:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879420.921335:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788879420.921335:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879420.921341:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788879420.921342:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.921346:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879420.921347:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879420.921355:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:3.0:1788879420.921355:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879420.921362:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1788879420.921362:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879420.921368:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879420.921369:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879420.921375:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1788879420.921380:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290161536:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788879420.921381:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879420.921392:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879420.921393:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879420.921399:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879420.921401:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879420.921406:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879420.921406:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879420.921415:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879420.921424:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879420.921424:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879420.921458:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290161664/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879420.921470:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879420.921475:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:1.0:1788879420.921480:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879420.921497:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879420.921502:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879420.921507:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879420.921512:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879420.921518:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879420.921521:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879420.921524:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879420.921528:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879420.921533:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10359 00000100:00080000:3.0:1788879420.921556:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1788879420.921559:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879420.921561:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879420.921564:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879420.921568:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879420.921570:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879420.921573:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879420.921576:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879420.921578:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879420.921580:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879420.921580:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879420.921587:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879420.921590:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.921593:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788879420.921593:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879420.921597:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879420.921599:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879420.921602:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.921604:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879420.921607:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879420.921612:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879420.921614:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788879420.921616:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879420.921621:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879420.921625:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879420.921627:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879420.921630:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879420.921631:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879420.921632:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305032745 02000000:00000001:3.0:1788879420.921634:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879420.921644:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879420.921649:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290161792/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879420.921650:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879420.921659:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879420.921661:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879420.921663:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879420.921665:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879420.921667:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879420.921668:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879420.921669:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879420.921672:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305032745 00000100:00000001:3.0:1788879420.921672:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879420.921678:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10359 00000100:00000001:1.0:1788879420.921684:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788879420.921684:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879420.921688:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879420.921691:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879420.921693:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788879420.921695:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879420.921699:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879420.921701:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879420.921704:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879420.921707:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1788879420.921709:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879420.921714:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879420.921717:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:1.0:1788879420.921723:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879420/real 1788879420] req@ffff94e50449e300 x1875765290161536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879420.921724:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879420.921730:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879420.921734:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879420.921739:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879420.921745:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879420.921751:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879420.921755:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879420.921763:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879420.921765:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879420.921768:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1788879420.921771:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000400:00000001:1.0:1788879420.921772:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788879420.921775:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879420.921779:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879420.921780:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:3.0:1788879420.921781:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879420.921785:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.921786:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879420.921791:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879420.921793:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.921795:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879420.921799:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879420.921802:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290161920/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879420.921807:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290161536/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879420.921845:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879420.921845:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879420.921853:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.921854:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879420.921860:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.921863:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.921869:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879420.921871:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.921875:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879420.921879:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879420.921885:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879420.921910:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879420.921914:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879420.921920:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290161536:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879420.921928:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.921966:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879420.921969:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879420.921972:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879420.921977:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879420.921980:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879420.921983:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879420.922007:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879420.922009:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879420.922013:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879420.922015:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922018:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922028:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879420.922031:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.922033:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879420.922035:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879420.922037:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879420.922039:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879420.922040:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879420.922043:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879420.922045:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879420.922047:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879420.922052:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290161664:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879420.922056:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879420.922058:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879420.922060:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879420.922062:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879420.922066:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.922082:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879420.922084:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:1.0:1788879420.922090:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879420.922104:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879420.922108:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879420.922110:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879420.922112:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.922113:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879420.922115:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.922120:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305032204 00000100:00000001:1.0:1788879420.922123:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305032745 00000100:00000001:1.0:1788879420.922125:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305032204 00000100:00000001:1.0:1788879420.922127:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305032745 00000100:00000001:1.0:1788879420.922128:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879420.922130:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.922131:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879420.922134:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879420/real 1788879420] req@ffff94e50449c700 x1875765290161664/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879420.922147:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879420.922148:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879420.922151:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879420.922153:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879420.922155:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922157:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879420.922159:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879420.922164:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290161664/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879420.922172:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922175:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879420.922177:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922179:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879420.922181:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922183:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879420.922202:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879420.922205:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879420.922209:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879420.922212:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879420.922215:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290161664:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879420.922220:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879420.922222:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879420.922223:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879420.922226:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879420.922228:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879420.922231:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879420.922234:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290161792:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879420.922238:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879420.922240:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879420.922243:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879420.922245:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879420.922248:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.922255:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879420.922257:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:1.0:1788879420.922262:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879420.922281:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879420.922288:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879420.922290:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879420.922292:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.922294:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879420.922297:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.922301:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305032202 00000100:00000001:1.0:1788879420.922304:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305032745 00000100:00000001:1.0:1788879420.922307:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305032202 00000100:00000001:1.0:1788879420.922310:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305032745 00000100:00000001:1.0:1788879420.922312:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879420.922314:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.922316:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879420.922321:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879420/real 1788879420] req@ffff94e50449f480 x1875765290161792/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879420.922330:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879420.922332:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879420.922336:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879420.922338:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879420.922339:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922341:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879420.922343:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879420.922348:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290161792/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879420.922357:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922361:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879420.922364:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922365:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879420.922366:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922367:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879420.922380:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879420.922398:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879420.922403:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879420.922405:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879420.922408:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290161792:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879420.922412:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879420.922414:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879420.922415:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879420.922418:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879420.922420:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879420.922422:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879420.922425:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290161920:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879420.922429:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879420.922431:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879420.922433:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879420.922434:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879420.922438:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.922445:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879420.922446:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:1.0:1788879420.922451:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879420.922465:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879420.922469:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879420.922471:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879420.922473:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.922475:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879420.922478:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.922482:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305032202 00000100:00000001:1.0:1788879420.922484:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305032745 00000100:00000001:1.0:1788879420.922488:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305032202 00000100:00000001:1.0:1788879420.922490:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305032745 00000100:00000001:1.0:1788879420.922492:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879420.922494:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.922496:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879420.922500:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879420/real 1788879420] req@ffff94e50449f800 x1875765290161920/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879420.922507:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879420.922508:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879420.922510:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879420.922512:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879420.922513:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922515:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879420.922517:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879420.922520:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290161920/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879420.922526:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922528:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879420.922530:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922532:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879420.922533:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922535:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879420.922549:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879420.922551:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879420.922555:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879420.922557:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879420.922560:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290161920:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879420.922564:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922566:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879420.922567:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879420.922569:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879420.922573:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879420.922575:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879420.922576:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879420.922579:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879420.922581:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879420.922583:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879420.922584:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922586:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879420.922587:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879420.922589:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879420.922591:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879420.922592:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879420.922593:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879420.922595:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879420.922596:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879420.922598:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879420.922599:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922600:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879420.922601:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879420.922602:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879420.922604:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879420.922605:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879420.922606:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879420.922609:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879420.922620:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879420.922623:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879420.922625:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922628:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.922634:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879420.922636:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.922638:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879420.922657:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879420.985065:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879420.985075:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879420.985077:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879420.985083:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879420.985088:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879420.985092:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.985095:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879420.985099:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879420.985103:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879421.688992:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879421.688998:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879421.689004:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879421.689005:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879421.689045:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879421.689048:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879421.689052:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879422.008974:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879422.008997:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879422.009000:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879422.009004:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879422.009007:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879422.009011:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879422.009031:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879422.009034:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879422.009038:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879422.713064:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879422.713072:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879422.713081:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879422.713084:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879422.713157:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879422.713162:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879422.713166:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879423.033023:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879423.033033:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879423.033037:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879423.033045:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879423.033050:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879423.033055:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879423.033059:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879423.033063:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879423.033067:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879423.737045:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879423.737052:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879423.737058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879423.737059:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879423.737090:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879423.737094:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879423.737099:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879424.057007:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879424.057014:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879424.057017:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879424.057021:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879424.057024:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879424.057029:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879424.057031:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879424.057034:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879424.057036:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879424.761041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879424.761047:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879424.761052:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879424.761054:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879424.761056:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879424.761059:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879424.761061:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879424.761062:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879424.761064:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879424.761065:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879424.761088:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879424.761090:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879424.761092:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879425.080997:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879425.081003:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879425.081005:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879425.081011:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879425.081015:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879425.081019:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879425.081022:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879425.081026:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879425.081028:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879425.785006:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879425.785012:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879425.785018:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879425.785019:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879425.785119:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879425.785122:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879425.785127:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879426.041106:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879426.041120:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879426.041124:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879426.041130:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879426.041137:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879426.041147:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10364 00000100:00080000:3.0:1788879426.041162:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879426.041184:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879426.041186:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879426.041194:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879426.041197:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879426.041201:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879426.041204:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879426.041206:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879426.041207:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879426.041212:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879426.041214:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879426.041218:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879426.041222:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879426.041225:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879426.041227:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879426.041230:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879426.041241:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879426.041243:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879426.041247:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879426.041250:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879426.041255:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879426.041267:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879426.041277:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290162048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879426.041287:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879426.041292:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879426.041307:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879426.041311:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879426.041314:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879426.041315:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879426.041317:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879426.041320:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879426.041322:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10364 00000100:00080000:3.0:1788879426.041325:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879426.041327:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879426.041328:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879426.041331:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879426.041333:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879426.041335:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879426.041337:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879426.041340:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879426.041341:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879426.041344:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879426.041347:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879426.041349:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879426.041353:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788879426.041355:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879426.041360:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788879426.041361:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.041365:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879426.041366:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879426.041369:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879426.041372:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879426.041376:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:3.0:1788879426.041380:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879426.041381:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879426.041388:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788879426.041389:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879426.041394:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879426.041397:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290162048:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788879426.041400:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879426.041403:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.041406:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879426.041407:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879426.041410:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879426.041412:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879426.041413:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879426.041416:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879426.041421:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879426.041421:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879426.041430:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879426.041431:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290162176/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879426.041441:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879426.041443:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879426.041446:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879426.041449:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879426.041450:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788879426.041452:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879426.041454:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:3.0:1788879426.041457:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879426.041461:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879426.041462:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879426.041468:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879426.041470:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10364 00000100:00080000:3.0:1788879426.041474:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879426.041476:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879426.041477:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879426.041497:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879426.041499:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879426.041502:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879426.041505:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879426.041507:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879426.041510:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879426.041516:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879426.041520:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879426.041522:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879426.041526:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879426.041531:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879426.041535:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879426.041536:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879426.041543:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879426.041547:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879426.041551:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.041556:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788879426.041558:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879426.041564:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.041566:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879426.041567:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879426.041573:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879426.041575:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879426.041578:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.041580:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879426.041588:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.041590:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.041598:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305037865 00000100:00100000:3.0:1788879426.041599:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290162304/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879426.041613:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879426.041615:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879426.041617:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879426.041624:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.041626:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305037865 00000100:00080000:3.0:1788879426.041628:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879426.041637:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788879426.041638:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879426.041645:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879426.041648:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879426.041649:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879426.041658:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879426.041660:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879426.041666:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1788879426.041667:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10364 00000100:00080000:3.0:1788879426.041676:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788879426.041681:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879426/real 1788879426] req@ffff94e50449d880 x1875765290162048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879426.041681:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879426.041685:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879426.041695:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879426.041700:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879426.041705:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879426.041715:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879426.041723:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879426.041728:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:1.0:1788879426.041736:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1788879426.041737:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000400:00000001:1.0:1788879426.041741:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879426.041744:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.041750:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:3.0:1788879426.041750:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879426.041758:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:3.0:1788879426.041759:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879426.041762:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879426.041766:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.041768:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879426.041772:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879426.041774:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1788879426.041778:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879426.041787:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290162048/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1788879426.041788:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879426.041796:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879426.041806:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879426.041809:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879426.041811:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.041816:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879426.041817:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879426.041824:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879426.041826:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879426.041829:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.041831:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879426.041836:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879426.041839:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.041840:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879426.041845:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879426.041853:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1788879426.041855:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290162432/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1788879426.041863:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879426.041871:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788879426.041883:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1788879426.041884:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290162048:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879426.041892:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.041895:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.041903:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879426.041905:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.041906:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879426.041911:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879426.042000:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879426.042004:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879426.042007:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879426.042019:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879426.042022:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879426.042026:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879426.042029:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042033:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042043:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879426.042065:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.042070:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879426.042074:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879426.042078:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879426.042081:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879426.042083:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879426.042086:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879426.042089:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879426.042092:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879426.042098:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290162176:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879426.042103:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879426.042106:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879426.042108:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879426.042110:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879426.042115:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.042124:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879426.042126:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:1.0:1788879426.042132:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879426.042152:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879426.042158:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879426.042162:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879426.042165:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.042168:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879426.042173:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.042197:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305037324 00000100:00000001:1.0:1788879426.042201:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305037865 00000100:00000001:1.0:1788879426.042204:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305037324 00000100:00000001:1.0:1788879426.042208:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305037865 00000100:00000001:1.0:1788879426.042211:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879426.042213:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.042216:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879426.042222:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879426/real 1788879426] req@ffff94e50449fb80 x1875765290162176/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879426.042246:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879426.042248:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879426.042252:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879426.042255:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879426.042256:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042259:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879426.042260:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879426.042265:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290162176/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879426.042272:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042274:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879426.042277:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042278:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879426.042280:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042283:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879426.042304:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879426.042309:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879426.042314:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879426.042318:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879426.042323:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290162176:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879426.042329:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879426.042331:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879426.042334:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879426.042337:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879426.042340:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879426.042342:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879426.042346:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290162304:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879426.042349:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879426.042351:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879426.042354:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879426.042355:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879426.042359:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.042365:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879426.042366:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:1.0:1788879426.042370:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879426.042382:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879426.042387:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879426.042389:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879426.042390:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.042392:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879426.042394:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.042398:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305037322 00000100:00000001:1.0:1788879426.042400:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305037865 00000100:00000001:1.0:1788879426.042402:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305037322 00000100:00000001:1.0:1788879426.042404:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305037865 00000100:00000001:1.0:1788879426.042407:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879426.042408:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.042410:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879426.042413:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879426/real 1788879426] req@ffff94e50449c380 x1875765290162304/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879426.042420:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879426.042421:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879426.042423:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879426.042424:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879426.042425:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042427:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879426.042429:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879426.042432:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290162304/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879426.042438:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042440:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879426.042443:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042444:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879426.042445:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042447:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879426.042462:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879426.042465:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879426.042468:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879426.042470:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879426.042472:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290162304:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879426.042475:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879426.042476:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879426.042477:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879426.042479:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879426.042481:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879426.042482:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879426.042485:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290162432:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879426.042488:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879426.042490:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879426.042491:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879426.042492:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879426.042495:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.042498:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879426.042499:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:1.0:1788879426.042502:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879426.042510:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879426.042513:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879426.042515:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879426.042516:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.042518:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879426.042520:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.042523:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305037322 00000100:00000001:1.0:1788879426.042525:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305037865 00000100:00000001:1.0:1788879426.042527:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305037322 00000100:00000001:1.0:1788879426.042529:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305037865 00000100:00000001:1.0:1788879426.042532:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879426.042533:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.042535:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879426.042538:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879426/real 1788879426] req@ffff94e50449d180 x1875765290162432/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879426.042544:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879426.042545:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879426.042547:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879426.042548:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879426.042549:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042550:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879426.042552:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879426.042555:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290162432/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879426.042561:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042563:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879426.042565:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042566:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879426.042567:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042569:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879426.042584:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879426.042586:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879426.042588:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879426.042590:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879426.042593:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290162432:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879426.042596:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042598:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879426.042600:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879426.042601:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879426.042603:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879426.042605:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879426.042606:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879426.042608:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879426.042609:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879426.042611:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879426.042613:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042614:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879426.042615:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879426.042617:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879426.042619:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879426.042620:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879426.042621:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879426.042623:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879426.042624:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879426.042625:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879426.042626:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042627:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879426.042628:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879426.042629:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879426.042630:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879426.042631:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879426.042632:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879426.042633:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879426.042634:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879426.042635:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879426.042636:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042638:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.042641:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879426.042642:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.042644:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879426.042645:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879426.105021:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879426.105027:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879426.105030:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879426.105036:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879426.105040:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879426.105044:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.105048:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879426.105051:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879426.105054:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879426.809079:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879426.809091:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879426.809094:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879426.809100:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879426.809107:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879426.809110:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879426.809114:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879426.809118:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879426.809126:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879426.809129:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879426.809218:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879426.809223:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879426.809227:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879427.129053:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879427.129060:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879427.129064:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879427.129070:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879427.129075:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879427.129080:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879427.129084:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879427.129087:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879427.129091:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879427.833043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879427.833050:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879427.833058:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879427.833060:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879427.833113:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879427.833117:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879427.833122:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879428.153083:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879428.153093:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879428.153097:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879428.153104:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879428.153109:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879428.153114:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879428.153117:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879428.153121:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879428.153124:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879428.857072:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879428.857080:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879428.857087:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879428.857089:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879428.857125:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879428.857129:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879428.857133:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879429.177055:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879429.177064:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879429.177067:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879429.177073:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879429.177077:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879429.177082:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879429.177085:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879429.177089:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879429.177092:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879429.881088:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879429.881101:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879429.881110:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879429.881112:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879429.881149:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879429.881154:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879429.881158:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879430.201047:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879430.201055:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879430.201058:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879430.201063:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879430.201067:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879430.201071:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879430.201074:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879430.201094:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879430.201097:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879430.905081:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879430.905089:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879430.905097:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879430.905098:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879430.905202:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879430.905210:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879430.905217:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879431.161015:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879431.161025:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879431.161028:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879431.161032:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879431.161037:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879431.161044:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10370 00000100:00080000:3.0:1788879431.161055:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879431.161059:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879431.161061:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879431.161068:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879431.161070:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879431.161074:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879431.161076:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879431.161078:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879431.161080:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879431.161083:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879431.161086:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879431.161088:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879431.161092:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879431.161094:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161096:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879431.161099:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879431.161110:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879431.161111:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161115:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879431.161117:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879431.161123:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161134:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879431.161143:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290162560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879431.161153:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879431.161158:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161185:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879431.161189:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879431.161200:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879431.161202:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879431.161203:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879431.161206:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879431.161208:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10370 00000100:00080000:3.0:1788879431.161212:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879431.161214:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879431.161216:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879431.161219:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879431.161220:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879431.161222:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879431.161224:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879431.161226:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879431.161227:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879431.161229:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879431.161231:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879431.161232:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879431.161237:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879431.161239:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161241:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879431.161245:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879431.161248:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161250:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879431.161254:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879431.161256:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161258:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879431.161260:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879431.161261:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161266:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.161267:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1788879431.161274:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290162688/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879431.161275:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879431.161280:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879431.161285:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879431.161288:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161290:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879431.161296:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879431.161298:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879431.161300:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879431.161303:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879431.161305:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879431.161309:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879431.161312:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879431.161317:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788879431.161318:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290162560:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1788879431.161320:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879431.161329:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879431.161330:0:178195:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:1.0:1788879431.161336:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1788879431.161337:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10370 02000000:00000001:1.0:1788879431.161340:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879431.161343:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788879431.161347:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879431.161348:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879431.161352:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879431.161363:0:178195:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1788879431.161364:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879431.161365:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879431.161369:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879431.161372:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879431.161374:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879431.161375:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879431.161378:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879431.161380:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879431.161383:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879431.161385:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879431.161387:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161388:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879431.161391:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879431.161393:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161395:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.161398:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788879431.161400:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879431.161402:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 02000000:00000001:3.0:1788879431.161404:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161409:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879431.161410:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788879431.161412:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879431.161416:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161422:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879431.161427:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290162816/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879431.161434:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879431.161436:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161439:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879431.161441:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879431.161444:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879431.161446:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879431.161447:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879431.161450:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000800:00000100:1.0:1788879431.161455:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1788879431.161456:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10370 00000100:00080000:3.0:1788879431.161464:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879431.161465:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879431.161466:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879431.161469:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879431.161470:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879431.161472:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879431.161473:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879431.161475:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879431.161476:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879431.161478:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879431.161480:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879431.161481:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879431.161483:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879431.161485:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161490:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879431.161491:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879431.161493:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161494:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879431.161498:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879431.161499:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161501:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879431.161502:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879431.161503:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161507:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879431.161510:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290162944/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879431.161517:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879431.161518:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879431.161520:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.161537:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879431.161544:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879431.161547:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.161549:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879431.161554:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.161558:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305042985 00000100:00000001:1.0:1788879431.161568:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788879431.161582:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305042985 00000100:00000001:1.0:1788879431.161586:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879431.161589:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879431.161591:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.161593:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879431.161601:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879431/real 1788879431] req@ffff94e50449df80 x1875765290162560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879431.161612:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879431.161614:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879431.161617:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879431.161619:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879431.161626:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.161628:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879431.161630:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879431.161634:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290162560/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879431.161652:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.161655:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879431.161658:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.161660:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879431.161661:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.161664:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879431.161666:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879431.161669:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879431.161674:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879431.161677:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879431.161681:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290162560:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879431.161685:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.161687:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879431.161688:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879431.161691:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879431.161694:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879431.161696:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879431.161697:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879431.161701:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879431.161703:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879431.161705:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879431.161706:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.161709:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.161716:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879431.161718:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.161721:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879431.161722:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879431.161725:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879431.161726:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879431.161728:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879431.161731:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879431.161734:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879431.161736:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879431.161742:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290162688:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879431.161746:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879431.161748:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879431.161749:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879431.161751:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879431.161754:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.161759:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879431.161761:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000100:00000001:1.0:1788879431.161765:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879431.161777:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879431.161782:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879431.161784:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879431.161785:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.161786:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879431.161789:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.161793:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305042443 00000100:00000001:1.0:1788879431.161796:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305042985 00000100:00000001:1.0:1788879431.161798:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305042443 00000100:00000001:1.0:1788879431.161800:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305042985 00000100:00000001:1.0:1788879431.161802:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879431.161803:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.161805:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879431.161808:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879431/real 1788879431] req@ffff94e50449e680 x1875765290162688/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879431.161817:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879431.161818:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879431.161820:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879431.161821:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879431.161822:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.161824:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879431.161825:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879431.161829:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290162688/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879431.161835:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.161837:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879431.161839:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.161840:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879431.161841:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.161842:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879431.161878:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879431.161880:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879431.161883:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879431.161884:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879431.161887:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290162688:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879431.161890:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879431.161891:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879431.161892:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879431.161895:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879431.161896:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879431.161898:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879431.161900:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290162816:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879431.161953:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879431.161955:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879431.161957:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879431.161958:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879431.161973:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.161979:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879431.161980:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:1.0:1788879431.161983:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879431.161995:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879431.162000:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879431.162002:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879431.162003:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.162004:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879431.162007:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.162010:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305042442 00000100:00000001:1.0:1788879431.162012:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305042985 00000100:00000001:1.0:1788879431.162014:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305042442 00000100:00000001:1.0:1788879431.162016:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305042985 00000100:00000001:1.0:1788879431.162018:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879431.162020:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.162021:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879431.162025:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879431/real 1788879431] req@ffff94e50449ce00 x1875765290162816/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879431.162031:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879431.162032:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879431.162034:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879431.162035:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879431.162036:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.162038:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879431.162039:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879431.162043:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290162816/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879431.162048:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.162050:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879431.162053:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.162054:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879431.162055:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.162057:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879431.162072:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879431.162074:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879431.162077:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879431.162079:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879431.162082:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290162816:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879431.162085:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879431.162086:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879431.162087:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879431.162088:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879431.162090:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879431.162092:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879431.162094:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290162944:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879431.162097:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879431.162098:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879431.162100:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879431.162101:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879431.162103:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.162107:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879431.162108:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:1.0:1788879431.162110:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879431.162117:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879431.162120:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879431.162122:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879431.162123:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.162124:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879431.162126:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.162129:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305042442 00000100:00000001:1.0:1788879431.162131:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305042985 00000100:00000001:1.0:1788879431.162133:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305042442 00000100:00000001:1.0:1788879431.162135:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305042985 00000100:00000001:1.0:1788879431.162137:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879431.162139:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.162140:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879431.162143:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879431/real 1788879431] req@ffff94e50449d180 x1875765290162944/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879431.162149:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879431.162150:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879431.162151:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879431.162152:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879431.162153:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.162155:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879431.162156:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879431.162159:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290162944/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879431.162164:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.162166:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879431.162180:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.162182:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879431.162183:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.162184:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879431.162197:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879431.162199:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879431.162201:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879431.162203:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879431.162206:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290162944:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879431.162209:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.162210:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879431.162212:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879431.162214:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879431.162216:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879431.162217:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879431.162218:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879431.162220:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879431.162221:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879431.162223:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879431.162224:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.162226:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879431.162227:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879431.162228:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879431.162229:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879431.162230:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879431.162231:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879431.162232:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879431.162233:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879431.162235:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879431.162236:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.162237:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879431.162238:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879431.162239:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879431.162240:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879431.162241:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879431.162242:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879431.162243:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879431.162244:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879431.162245:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879431.162246:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.162248:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.162250:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879431.162251:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.162253:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879431.162254:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879431.225026:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879431.225036:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879431.225039:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879431.225046:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879431.225051:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879431.225055:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.225059:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879431.225062:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879431.225066:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879431.929060:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879431.929072:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879431.929082:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879431.929085:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879431.929122:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879431.929126:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879431.929130:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879432.249076:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879432.249084:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879432.249087:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879432.249093:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879432.249098:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879432.249104:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879432.249108:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879432.249111:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879432.249114:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879432.953007:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879432.953014:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879432.953020:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879432.953022:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879432.953044:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879432.953047:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879432.953051:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879433.272980:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879433.272986:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879433.272988:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879433.272992:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879433.272995:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879433.272999:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879433.273001:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879433.273004:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879433.273006:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879433.977018:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879433.977023:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879433.977028:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879433.977029:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879433.977070:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879433.977073:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879433.977076:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879434.297023:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879434.297034:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879434.297036:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879434.297043:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879434.297048:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879434.297052:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879434.297055:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879434.297059:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879434.297062:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879435.001013:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879435.001021:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879435.001027:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879435.001031:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879435.001034:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879435.001037:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879435.001040:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879435.001042:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879435.001044:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879435.001047:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879435.001113:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879435.001121:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879435.001126:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879435.320973:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879435.320981:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879435.320982:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879435.320987:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879435.320990:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879435.320994:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879435.320996:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879435.320999:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879435.321002:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879436.024976:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879436.024990:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879436.024995:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879436.024996:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879436.025027:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879436.025031:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879436.025034:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879436.281110:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879436.281124:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879436.281129:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879436.281134:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879436.281141:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879436.281151:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10375 00000100:00080000:0.0:1788879436.281166:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879436.281170:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879436.281173:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879436.281185:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879436.281189:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879436.281194:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879436.281197:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879436.281200:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879436.281203:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879436.281209:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879436.281213:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879436.281217:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879436.281222:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879436.281226:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879436.281229:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879436.281232:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879436.281249:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879436.281251:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879436.281255:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879436.281258:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879436.281265:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879436.281283:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879436.281296:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290163072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879436.281310:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879436.281317:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879436.281342:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879436.281347:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879436.281351:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879436.281353:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879436.281356:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879436.281360:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879436.281363:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10375 00000100:00080000:0.0:1788879436.281368:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879436.281371:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879436.281373:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879436.281376:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879436.281378:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879436.281381:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879436.281384:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879436.281389:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879436.281392:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879436.281398:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879436.281402:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879436.281404:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879436.281413:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879436.281415:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788879436.281417:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879436.281423:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879436.281424:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00100000:0.0:1788879436.281429:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879436.281431:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00000001:0.0:1788879436.281440:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.281441:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879436.281448:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879436.281450:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879436.281460:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879436.281461:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879436.281468:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879436.281469:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879436.281477:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879436.281480:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879436.281482:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879436.281488:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879436.281492:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290163072:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879436.281498:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.281503:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1788879436.281512:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290163200/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879436.281524:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879436.281526:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788879436.281527:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879436.281533:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879436.281537:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879436.281539:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1788879436.281544:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879436.281551:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879436.281552:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879436.281560:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879436.281563:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879436.281566:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879436.281571:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10375 00000100:00080000:0.0:1788879436.281576:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879436.281578:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879436.281580:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879436.281583:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1788879436.281586:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879436.281588:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000020:00000001:0.0:1788879436.281592:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879436.281598:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879436.281598:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1788879436.281607:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879436.281611:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879436.281613:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879436.281618:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879436.281621:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879436.281624:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879436.281628:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879436.281631:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879436.281633:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879436.281636:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879436.281639:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879436.281643:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879436.281650:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879436.281652:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879436.281654:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879436.281656:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879436.281661:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879436.281663:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879436.281670:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879436.281681:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290163328/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879436.281684:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879436.281696:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879436.281698:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879436.281705:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.281706:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879436.281710:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.281717:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:0.0:1788879436.281718:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879436.281729:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879436.281732:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879436.281735:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879436.281737:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879436.281749:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305048105 00000100:00000001:0.0:1788879436.281750:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879436.281761:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10375 00000100:00080000:0.0:1788879436.281773:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879436.281775:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788879436.281779:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879436.281786:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879436.281792:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305048105 00000020:00000001:0.0:1788879436.281795:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879436.281803:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879436.281804:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:0.0:1788879436.281812:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879436.281815:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879436.281825:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879436.281829:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879436.281831:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879436.281839:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1788879436.281845:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879436/real 1788879436] req@ffff94e508978e00 x1875765290163072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879436.281846:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879436.281860:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879436.281868:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879436.281875:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879436.281976:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:1.0:1788879436.281983:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1788879436.281984:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879436.281990:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879436.281992:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879436.281998:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879436.282002:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879436.282004:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879436.282012:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879436.282015:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282017:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879436.282019:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879436.282025:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879436.282026:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:0.0:1788879436.282031:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282032:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282044:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879436.282048:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879436.282050:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879436.282060:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290163072/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879436.282069:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290163456/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879436.282101:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879436.282111:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879436.282113:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879436.282119:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282122:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282127:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879436.282129:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282131:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282139:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879436.282142:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879436.282147:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879436.282155:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879436.282167:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879436.282173:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290163072:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879436.282180:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282183:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879436.282186:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879436.282189:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879436.282195:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879436.282198:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879436.282201:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879436.282206:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879436.282208:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879436.282213:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879436.282215:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282218:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282229:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879436.282231:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282234:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879436.282236:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879436.282239:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879436.282241:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879436.282243:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879436.282246:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879436.282248:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879436.282251:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879436.282255:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290163200:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879436.282260:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879436.282263:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879436.282265:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879436.282267:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879436.282272:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282281:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879436.282282:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:1.0:1788879436.282288:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879436.282308:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879436.282315:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879436.282317:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879436.282320:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282322:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879436.282326:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282332:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305047564 00000100:00000001:1.0:1788879436.282336:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305048105 00000100:00000001:1.0:1788879436.282339:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305047564 00000100:00000001:1.0:1788879436.282342:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305048105 00000100:00000001:1.0:1788879436.282344:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879436.282347:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282349:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879436.282355:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879436/real 1788879436] req@ffff94e50897ad80 x1875765290163200/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879436.282375:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879436.282377:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879436.282380:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879436.282382:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879436.282384:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282386:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879436.282388:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879436.282393:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290163200/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879436.282402:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282404:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879436.282407:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282409:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879436.282411:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282413:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879436.282429:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879436.282432:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879436.282436:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879436.282438:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879436.282442:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290163200:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879436.282446:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879436.282448:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879436.282450:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879436.282453:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879436.282455:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879436.282457:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879436.282461:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290163328:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879436.282464:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879436.282466:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879436.282469:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879436.282471:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879436.282475:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282482:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879436.282484:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:1.0:1788879436.282489:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879436.282505:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879436.282512:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879436.282515:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879436.282517:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282519:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879436.282522:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282526:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305047562 00000100:00000001:1.0:1788879436.282529:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305048105 00000100:00000001:1.0:1788879436.282532:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305047562 00000100:00000001:1.0:1788879436.282535:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305048105 00000100:00000001:1.0:1788879436.282538:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879436.282540:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282542:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879436.282546:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879436/real 1788879436] req@ffff94e508979c00 x1875765290163328/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879436.282556:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879436.282557:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879436.282560:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879436.282562:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879436.282564:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282566:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879436.282568:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879436.282575:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290163328/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879436.282584:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282587:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879436.282593:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282596:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879436.282598:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282601:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879436.282614:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879436.282618:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879436.282622:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879436.282625:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879436.282629:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290163328:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879436.282633:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879436.282634:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879436.282636:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879436.282638:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879436.282640:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879436.282643:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879436.282646:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290163456:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879436.282649:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879436.282651:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879436.282653:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879436.282655:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879436.282659:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282664:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879436.282665:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:1.0:1788879436.282669:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879436.282680:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879436.282685:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879436.282687:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879436.282689:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282691:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879436.282694:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282698:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305047562 00000100:00000001:1.0:1788879436.282701:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305048105 00000100:00000001:1.0:1788879436.282704:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305047562 00000100:00000001:1.0:1788879436.282707:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305048105 00000100:00000001:1.0:1788879436.282710:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879436.282712:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282715:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879436.282719:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879436/real 1788879436] req@ffff94e50897a300 x1875765290163456/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879436.282728:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879436.282729:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879436.282732:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879436.282733:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879436.282735:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282737:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879436.282738:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879436.282743:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290163456/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879436.282751:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282753:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879436.282756:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282758:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879436.282760:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282762:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879436.282775:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879436.282778:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879436.282781:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879436.282784:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879436.282788:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290163456:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879436.282793:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282795:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879436.282797:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879436.282799:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879436.282802:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879436.282804:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879436.282805:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879436.282809:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879436.282811:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879436.282815:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879436.282817:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282819:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879436.282821:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879436.282823:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879436.282825:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879436.282827:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879436.282828:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879436.282831:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879436.282832:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879436.282834:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879436.282836:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282838:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879436.282839:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879436.282840:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879436.282842:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879436.282843:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879436.282844:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879436.282847:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879436.282849:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879436.282851:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879436.282854:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282857:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.282862:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879436.282864:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.282868:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879436.282871:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879436.345045:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879436.345052:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879436.345054:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879436.345061:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879436.345065:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879436.345069:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.345072:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879436.345075:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879436.345079:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879437.048969:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879437.048974:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879437.048976:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879437.048980:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879437.048984:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879437.048986:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879437.048988:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879437.048989:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879437.048993:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879437.048994:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879437.049009:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879437.049011:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879437.049013:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879437.369037:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879437.369043:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879437.369045:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879437.369049:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879437.369051:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879437.369054:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879437.369057:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879437.369059:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879437.369061:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879438.073073:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879438.073084:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879438.073091:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879438.073092:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879438.073131:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879438.073135:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879438.073139:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879438.393059:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879438.393069:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879438.393072:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879438.393078:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879438.393083:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879438.393088:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879438.393092:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879438.393096:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879438.393100:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879439.097077:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879439.097088:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879439.097095:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879439.097097:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879439.097255:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879439.097259:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879439.097263:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879439.417051:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879439.417060:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879439.417062:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879439.417069:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879439.417073:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879439.417078:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879439.417081:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879439.417085:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879439.417088:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879440.121155:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879440.121164:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879440.121171:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879440.121172:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879440.121298:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879440.121302:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879440.121308:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879440.441084:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879440.441093:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879440.441097:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879440.441104:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879440.441108:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879440.441113:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879440.441116:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879440.441121:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879440.441124:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879441.145005:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879441.145010:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879441.145015:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879441.145016:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879441.145036:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879441.145039:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879441.145042:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879441.401060:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879441.401075:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879441.401079:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879441.401084:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879441.401092:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879441.401103:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10380 00000100:00080000:2.0:1788879441.401119:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879441.401123:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879441.401126:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879441.401148:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879441.401153:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879441.401159:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879441.401162:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879441.401165:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879441.401167:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879441.401173:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879441.401176:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879441.401180:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879441.401185:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879441.401189:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879441.401192:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879441.401206:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879441.401223:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879441.401225:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879441.401230:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879441.401234:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879441.401240:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879441.401257:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879441.401270:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290163584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879441.401285:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879441.401293:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879441.401319:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879441.401324:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879441.401328:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879441.401330:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879441.401333:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879441.401337:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879441.401350:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10380 00000100:00080000:2.0:1788879441.401355:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879441.401358:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879441.401362:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879441.401367:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879441.401370:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879441.401373:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879441.401376:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879441.401381:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879441.401382:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879441.401385:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879441.401392:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788879441.401395:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879441.401398:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879441.401400:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.401404:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788879441.401405:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879441.401419:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879441.401421:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879441.401424:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879441.401435:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879441.401438:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:2.0:1788879441.401443:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879441.401445:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1788879441.401453:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879441.401455:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879441.401458:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879441.401465:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879441.401469:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879441.401472:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1788879441.401473:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290163584:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879441.401475:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879441.401479:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.401488:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788879441.401492:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879441.401494:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879441.401500:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879441.401509:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290163712/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879441.401511:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879441.401543:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879441.401544:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879441.401554:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879441.401567:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.401571:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788879441.401573:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879441.401576:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:1.0:1788879441.401585:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788879441.401603:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879441.401609:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879441.401613:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879441.401621:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879441.401627:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10380 00000100:00080000:2.0:1788879441.401638:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879441.401646:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000800:00000100:1.0:1788879441.401647:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879441.401655:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879441.401678:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879441.401683:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879441.401689:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:2.0:1788879441.401690:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879441.401698:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1788879441.401700:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879441.401704:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879441.401711:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879441.401712:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879441.401722:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879441.401728:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.401757:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305053225 00000100:00080000:2.0:1788879441.401758:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879441.401769:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.401772:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1788879441.401776:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879441.401784:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879441.401788:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879441.401792:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879441.401793:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305053225 00000008:00100000:2.0:1788879441.401796:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879441.401800:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000008:00000001:2.0:1788879441.401802:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879441.401807:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.401808:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879441.401818:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.401822:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1788879441.401822:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879441.401869:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879441.401872:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879441/real 1788879441] req@ffff94e5045e5500 x1875765290163584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879441.401877:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879441.401881:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879441.401888:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879441.401943:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879441.401945:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879441.401952:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879441.401955:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879441.401957:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.401959:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879441.401961:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.401963:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879441.401978:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290163840/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879441.401995:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290163584/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879441.402029:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879441.402037:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879441.402039:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879441.402057:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402061:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879441.402062:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879441.402078:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879441.402082:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879441.402083:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879441.402088:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879441.402093:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879441.402095:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879441.402103:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879441.402105:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10380 00000100:00000001:1.0:1788879441.402119:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1788879441.402119:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879441.402126:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879441.402128:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879441.402133:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879441.402138:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879441.402143:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1788879441.402143:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879441.402153:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879441.402172:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879441.402176:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879441.402178:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879441.402184:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879441.402190:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879441.402196:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:1.0:1788879441.402198:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 02000000:00000001:2.0:1788879441.402202:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879441.402207:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402208:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788879441.402211:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879441.402221:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879441.402225:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290163584:192.168.203.129@tcp:250:kworker.0 00000008:00000001:2.0:1788879441.402226:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879441.402240:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402244:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402247:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788879441.402248:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879441.402253:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879441.402254:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879441.402259:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879441.402260:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879441.402272:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879441.402275:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:2.0:1788879441.402276:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879441.402281:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879441.402285:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:2.0:1788879441.402287:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879441.402292:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879441.402300:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00100000:2.0:1788879441.402301:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290163968/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879441.402308:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879441.402313:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402326:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879441.402341:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879441.402357:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879441.402362:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402364:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879441.402368:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402375:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879441.402382:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879441.402384:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879441.402387:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879441.402389:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879441.402393:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879441.402396:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879441.402399:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879441.402406:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290163712:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879441.402411:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879441.402414:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879441.402415:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879441.402417:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879441.402421:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402428:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879441.402430:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:1.0:1788879441.402434:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879441.402451:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879441.402456:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879441.402458:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879441.402459:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402461:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879441.402464:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402468:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305052684 00000100:00000001:1.0:1788879441.402471:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305053225 00000100:00000001:1.0:1788879441.402473:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305052684 00000100:00000001:1.0:1788879441.402475:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305053225 00000100:00000001:1.0:1788879441.402477:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879441.402479:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402481:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879441.402485:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879441/real 1788879441] req@ffff94e5045e5c00 x1875765290163712/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879441.402500:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879441.402501:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879441.402503:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879441.402505:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879441.402506:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402508:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879441.402510:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879441.402513:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290163712/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879441.402519:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402521:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879441.402524:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402525:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879441.402526:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402527:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879441.402545:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879441.402547:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879441.402550:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879441.402552:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879441.402555:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290163712:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879441.402559:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879441.402560:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879441.402562:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879441.402565:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879441.402566:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879441.402568:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879441.402571:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290163840:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879441.402589:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879441.402590:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879441.402592:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879441.402593:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879441.402595:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402599:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879441.402600:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:1.0:1788879441.402603:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879441.402610:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879441.402613:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879441.402614:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879441.402615:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402616:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879441.402618:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402620:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305052682 00000100:00000001:1.0:1788879441.402622:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305053225 00000100:00000001:1.0:1788879441.402624:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305052682 00000100:00000001:1.0:1788879441.402626:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305053225 00000100:00000001:1.0:1788879441.402627:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879441.402628:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402629:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879441.402631:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879441/real 1788879441] req@ffff94e5045e7100 x1875765290163840/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879441.402636:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879441.402654:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879441.402657:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879441.402658:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879441.402659:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402661:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879441.402662:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879441.402665:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290163840/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879441.402671:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402673:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879441.402675:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402677:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879441.402678:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402680:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879441.402695:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879441.402698:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879441.402701:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879441.402703:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879441.402706:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290163840:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879441.402709:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879441.402710:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879441.402711:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879441.402714:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879441.402715:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879441.402717:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879441.402719:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290163968:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879441.402723:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879441.402724:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879441.402726:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879441.402728:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879441.402730:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402734:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879441.402735:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:1.0:1788879441.402739:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879441.402766:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879441.402769:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879441.402771:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879441.402773:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402774:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879441.402776:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402779:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305052682 00000100:00000001:1.0:1788879441.402781:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305053225 00000100:00000001:1.0:1788879441.402784:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305052682 00000100:00000001:1.0:1788879441.402786:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305053225 00000100:00000001:1.0:1788879441.402788:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879441.402789:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402791:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879441.402794:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879441/real 1788879441] req@ffff94e5045e4a80 x1875765290163968/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879441.402800:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879441.402801:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879441.402802:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879441.402803:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879441.402804:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402806:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879441.402807:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879441.402810:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290163968/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879441.402815:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402817:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879441.402835:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402836:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879441.402837:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402838:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879441.402869:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879441.402872:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879441.402876:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879441.402878:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879441.402882:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290163968:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879441.402886:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402888:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879441.402890:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879441.402892:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879441.402894:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879441.402896:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879441.402897:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879441.402899:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879441.402900:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879441.402919:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879441.402921:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402922:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879441.402923:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879441.402924:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879441.402925:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879441.402926:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879441.402927:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879441.402928:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879441.402929:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879441.402930:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879441.402931:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402932:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879441.402933:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879441.402934:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879441.402935:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879441.402936:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879441.402937:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879441.402937:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879441.402938:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879441.402939:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879441.402940:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402941:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.402943:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879441.402944:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.402945:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879441.402946:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879441.465157:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879441.465166:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879441.465170:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879441.465178:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879441.465183:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879441.465189:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.465193:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879441.465214:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879441.465220:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879442.169036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879442.169042:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879442.169048:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879442.169050:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879442.169072:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879442.169075:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879442.169078:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879442.489015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879442.489022:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879442.489025:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879442.489031:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879442.489035:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879442.489039:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879442.489042:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879442.489045:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879442.489048:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879443.193041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879443.193050:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879443.193055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879443.193056:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879443.193144:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879443.193150:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879443.193155:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879443.513066:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879443.513078:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879443.513080:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879443.513087:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879443.513093:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879443.513098:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879443.513102:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879443.513106:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879443.513110:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879444.216980:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879444.216985:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879444.216990:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879444.216993:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879444.216995:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879444.216999:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879444.217001:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879444.217003:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879444.217004:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879444.217006:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879444.217024:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879444.217027:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879444.217029:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879444.537082:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879444.537091:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879444.537094:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879444.537101:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879444.537106:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879444.537111:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879444.537115:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879444.537118:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879444.537122:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879445.241028:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879445.241035:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879445.241042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879445.241044:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879445.241072:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879445.241075:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879445.241079:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879445.561031:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879445.561039:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879445.561042:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879445.561047:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879445.561051:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879445.561055:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879445.561057:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879445.561060:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879445.561063:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879446.265056:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879446.265064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879446.265070:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879446.265072:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879446.265150:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879446.265155:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879446.265161:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879446.521118:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879446.521161:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879446.521165:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879446.521171:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879446.521180:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879446.521192:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10385 00000100:00080000:3.0:1788879446.521217:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879446.521223:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879446.521239:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879446.521268:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879446.521272:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879446.521277:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879446.521281:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879446.521284:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879446.521287:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879446.521294:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879446.521307:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879446.521311:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879446.521317:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879446.521321:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879446.521325:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879446.521329:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879446.521341:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879446.521344:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879446.521348:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879446.521352:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879446.521358:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879446.521381:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879446.521393:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290164096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879446.521404:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879446.521408:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879446.521427:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879446.521431:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879446.521435:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879446.521437:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879446.521439:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879446.521442:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879446.521447:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10385 00000100:00080000:3.0:1788879446.521453:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879446.521456:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879446.521460:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879446.521465:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879446.521467:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879446.521470:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879446.521476:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788879446.521477:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879446.521487:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879446.521489:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879446.521490:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879446.521496:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879446.521500:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879446.521502:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879446.521510:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879446.521516:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879446.521519:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879446.521528:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290164096:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879446.521534:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879446.521539:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879446.521542:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879446.521545:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879446.521554:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879446.521571:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879446.521574:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.521577:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788879446.521577:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879446.521585:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 02000000:00000001:3.0:1788879446.521591:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879446.521593:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879446.521668:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879446.521703:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879446.521711:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879446.521716:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.521720:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879446.521726:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.521731:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305058345 00000100:00000001:2.0:1788879446.521761:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788879446.521772:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.521773:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305058345 00000100:00000001:3.0:1788879446.521783:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879446.521784:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879446.521798:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879446.521803:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.521808:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879446.521849:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879446/real 1788879446] req@ffff94e50449c380 x1875765290164096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788879446.521860:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879446.521872:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879446.521899:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879446.521907:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879446.521912:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879446.521917:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:2.0:1788879446.521968:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1788879446.521970:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879446.521989:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879446.521997:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879446.522003:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.522009:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879446.522016:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00100000:3.0:1788879446.522019:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290164224/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879446.522021:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.522032:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879446.522039:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879446.522077:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879446.522085:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879446.522086:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290164096/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879446.522091:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879446.522105:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879446.522149:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879446.522152:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879446.522154:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879446.522159:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879446.522159:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879446.522176:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879446.522177:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879446.522180:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879446.522186:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788879446.522187:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10385 00000100:00080000:3.0:1788879446.522215:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879446.522217:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879446.522229:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879446.522230:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879446.522240:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879446.522241:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:3.0:1788879446.522257:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00080000:2.0:1788879446.522258:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000020:00000001:3.0:1788879446.522261:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879446.522263:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879446.522282:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290164096:192.168.203.129@tcp:250:kworker.0 00000020:00000001:3.0:1788879446.522284:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879446.522295:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879446.522299:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879446.522301:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879446.522309:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879446.522311:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879446.522313:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879446.522318:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879446.522320:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.522322:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879446.522324:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879446.522327:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00100000:3.0:1788879446.522329:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879446.522330:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879446.522346:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000008:00000001:3.0:1788879446.522352:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879446.522353:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879446.522363:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:3.0:1788879446.522363:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879446.522374:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879446.522376:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879446.522381:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879446.522384:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879446.522387:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879446.522387:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879446.522398:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:3.0:1788879446.522415:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.522417:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879446.522425:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879446.522425:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.522439:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879446.522441:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290164352/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879446.522451:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879446.522453:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879446.522455:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879446.522458:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.522463:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879446.522477:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879446.522480:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879446.522484:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879446.522487:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879446.522488:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879446.522509:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1788879446.522510:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879446.522516:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879446.522520:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000001:2.0:1788879446.522522:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879446.522524:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879446.522536:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788879446.522539:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290164224:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879446.522542:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879446.522552:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788879446.522558:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10385 02000000:00000001:2.0:1788879446.522560:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879446.522565:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879446.522566:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1788879446.522578:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879446.522578:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879446.522592:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879446.522593:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879446.522598:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879446.522601:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879446.522604:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879446.522607:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1788879446.522608:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879446.522611:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:3.0:1788879446.522613:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879446.522618:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879446.522624:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879446.522625:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879446.522630:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879446.522633:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879446.522639:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879446.522642:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879446.522645:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879446.522655:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1788879446.522656:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00000001:3.0:1788879446.522669:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.522675:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879446.522676:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.522686:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879446.522691:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879446.522691:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879446.522699:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879446.522700:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.522703:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879446.522709:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879446.522712:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305057804 02000000:00000001:3.0:1788879446.522716:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879446.522717:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305058345 00000100:00000001:2.0:1788879446.522724:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305057804 02000000:00000001:3.0:1788879446.522727:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.522728:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305058345 00000100:00000001:2.0:1788879446.522740:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879446.522743:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.522744:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879446.522751:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879446.522754:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879446/real 1788879446] req@ffff94e50449fb80 x1875765290164224/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879446.522769:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879446.522771:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:3.0:1788879446.522773:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290164480/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879446.522774:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879446.522781:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879446.522785:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.522798:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879446.522799:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879446.522836:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290164224/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879446.522840:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879446.522848:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879446.522858:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.522881:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.522889:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879446.522894:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.522897:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879446.522950:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.522955:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879446.522973:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879446.522977:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879446.522983:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879446.522986:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879446.522990:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290164224:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879446.522994:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.522996:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879446.522998:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879446.522999:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879446.523002:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879446.523004:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879446.523005:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879446.523008:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879446.523009:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879446.523011:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879446.523012:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.523014:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.523017:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879446.523019:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.523021:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879446.523023:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879446.523025:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879446.523027:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879446.523028:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879446.523030:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879446.523033:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879446.523035:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879446.523037:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290164352:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879446.523040:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879446.523043:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879446.523044:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879446.523046:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879446.523048:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.523056:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879446.523057:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:2.0:1788879446.523062:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879446.523085:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879446.523089:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879446.523091:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879446.523092:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.523094:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879446.523096:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.523099:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305057803 00000100:00000001:2.0:1788879446.523102:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305058345 00000100:00000001:2.0:1788879446.523104:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305057803 00000100:00000001:2.0:1788879446.523106:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305058345 00000100:00000001:2.0:1788879446.523108:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879446.523109:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.523111:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879446.523114:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879446/real 1788879446] req@ffff94e50449d880 x1875765290164352/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879446.523121:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879446.523122:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879446.523124:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879446.523125:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879446.523127:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.523139:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879446.523140:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879446.523143:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290164352/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879446.523149:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.523150:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879446.523152:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.523153:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879446.523154:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.523156:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879446.523167:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879446.523168:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879446.523171:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879446.523173:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879446.523175:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290164352:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879446.523178:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879446.523179:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879446.523181:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879446.523183:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879446.523184:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879446.523186:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879446.523188:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290164480:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879446.523191:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879446.523192:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879446.523194:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879446.523195:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879446.523197:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.523200:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879446.523202:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:2.0:1788879446.523205:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879446.523213:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879446.523216:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879446.523218:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879446.523220:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.523221:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879446.523223:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.523226:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305057803 00000100:00000001:2.0:1788879446.523228:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305058345 00000100:00000001:2.0:1788879446.523230:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305057803 00000100:00000001:2.0:1788879446.523232:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305058345 00000100:00000001:2.0:1788879446.523234:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879446.523235:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.523236:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879446.523239:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879446/real 1788879446] req@ffff94e50449f800 x1875765290164480/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879446.523245:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879446.523246:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879446.523247:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879446.523249:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879446.523250:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.523251:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879446.523252:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879446.523254:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290164480/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879446.523268:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.523269:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879446.523271:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.523273:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879446.523274:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.523275:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879446.523286:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879446.523288:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879446.523291:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879446.523293:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879446.523295:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290164480:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879446.523299:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.523300:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879446.523301:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879446.523302:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879446.523303:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879446.523304:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879446.523305:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879446.523307:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879446.523308:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879446.523309:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879446.523311:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.523312:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879446.523313:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879446.523314:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879446.523316:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879446.523317:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879446.523318:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879446.523319:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879446.523320:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879446.523321:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879446.523322:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.523323:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879446.523325:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879446.523327:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879446.523328:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879446.523329:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879446.584985:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879446.584993:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879446.584995:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879446.585000:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879446.585004:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879446.585008:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879446.585011:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879446.585014:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879446.585016:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879447.288974:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879447.288980:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879447.288982:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879447.288985:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879447.288989:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879447.288991:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879447.288992:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879447.288994:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879447.288997:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879447.288998:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879447.289014:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879447.289016:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879447.289019:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879447.609038:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879447.609047:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879447.609050:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879447.609056:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879447.609060:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879447.609066:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879447.609069:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879447.609072:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879447.609075:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879448.313037:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879448.313061:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879448.313068:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879448.313069:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879448.313092:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879448.313094:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879448.313098:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879448.632992:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879448.633001:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879448.633004:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879448.633010:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879448.633015:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879448.633019:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879448.633023:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879448.633026:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879448.633029:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879449.337010:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879449.337018:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879449.337025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879449.337027:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879449.337055:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879449.337058:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879449.337062:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879449.657028:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879449.657039:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879449.657042:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879449.657050:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879449.657054:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879449.657059:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879449.657062:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879449.657065:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879449.657068:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879450.361025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879450.361033:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879450.361041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879450.361043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879450.361070:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879450.361073:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879450.361077:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879450.680992:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879450.681001:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879450.681004:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879450.681029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879450.681033:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879450.681038:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879450.681040:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879450.681044:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879450.681047:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879451.385060:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879451.385068:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879451.385076:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879451.385078:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879451.385107:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879451.385110:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879451.385114:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879451.641115:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879451.641127:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879451.641131:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879451.641136:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879451.641142:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879451.641152:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10390 00000100:00080000:3.0:1788879451.641167:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879451.641171:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879451.641174:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879451.641185:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879451.641188:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879451.641193:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879451.641197:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879451.641200:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879451.641202:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879451.641209:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879451.641212:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879451.641217:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879451.641222:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879451.641226:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.641229:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879451.641232:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879451.641246:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879451.641248:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.641252:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879451.641256:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879451.641263:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.641277:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879451.641289:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290164608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879451.641304:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879451.641310:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.641332:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879451.641337:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879451.641342:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879451.641344:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879451.641346:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879451.641350:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879451.641354:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10390 00000100:00080000:3.0:1788879451.641370:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879451.641373:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879451.641375:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879451.641380:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879451.641382:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879451.641387:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788879451.641387:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879451.641390:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879451.641397:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879451.641399:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879451.641401:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879451.641410:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1788879451.641411:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879451.641421:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.641422:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788879451.641424:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879451.641433:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879451.641436:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879451.641445:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879451.641449:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879451.641449:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879451.641458:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788879451.641461:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1788879451.641465:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879451.641477:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.641478:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879451.641489:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879451.641495:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879451.641497:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290164608:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879451.641499:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879451.641505:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879451.641511:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.641514:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879451.641519:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879451.641522:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879451.641528:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879451.641536:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1788879451.641536:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290164736/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788879451.641548:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.641553:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879451.641559:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.641565:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879451.641569:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879451.641573:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879451.641575:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879451.641576:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879451.641584:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00080000:3.0:1788879451.641585:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879451.641596:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879451.641597:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879451.641601:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10390 00000100:00080000:3.0:1788879451.641608:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879451.641610:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879451.641613:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879451.641616:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879451.641619:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879451.641621:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879451.641624:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879451.641626:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879451.641628:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879451.641632:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879451.641636:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879451.641639:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879451.641643:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879451.641645:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.641648:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879451.641652:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879451.641656:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.641658:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879451.641665:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879451.641667:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.641674:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000100:2.0:1788879451.641675:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879451.641677:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879451.641684:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.641691:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.641698:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00100000:3.0:1788879451.641701:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290164864/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879451.641707:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879451.641714:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.641724:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879451.641725:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879451.641734:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.641737:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.641739:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879451.641747:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879451.641748:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305063465 00000100:00080000:3.0:1788879451.641756:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879451.641762:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879451.641764:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879451.641767:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879451.641768:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788879451.641773:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10390 00000100:00080000:3.0:1788879451.641807:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879451.641810:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879451.641812:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879451.641816:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879451.641819:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305063465 00000020:00000001:3.0:1788879451.641820:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879451.641826:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879451.641833:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879451.641836:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879451.641840:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879451.641840:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879451.641851:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879451.641855:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879451.641856:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879451.641864:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879451.641871:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.641878:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1788879451.641880:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879451.641887:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.641892:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:2.0:1788879451.641893:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879451/real 1788879451] req@ffff94e50449f480 x1875765290164608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788879451.641896:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879451.641940:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.641943:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879451.641955:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879451.641961:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.641966:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:2.0:1788879451.641968:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1788879451.641969:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:2.0:1788879451.641977:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788879451.641978:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.641988:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879451.641990:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.641998:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879451.642004:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879451.642006:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290164992/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879451.642010:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879451.642016:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879451.642029:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1788879451.642032:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290164608/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879451.642035:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879451.642045:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.642064:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642072:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879451.642080:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642083:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879451.642086:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642091:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879451.642096:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879451.642115:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879451.642125:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879451.642130:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879451.642138:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290164608:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879451.642148:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642153:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879451.642156:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879451.642161:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879451.642166:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879451.642169:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879451.642171:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879451.642176:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879451.642178:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879451.642182:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879451.642184:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642187:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642196:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879451.642198:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.642201:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879451.642204:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879451.642206:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879451.642208:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879451.642210:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879451.642213:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879451.642216:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879451.642219:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879451.642224:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290164736:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879451.642228:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879451.642231:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879451.642234:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879451.642236:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879451.642241:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.642250:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879451.642252:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:2.0:1788879451.642258:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879451.642280:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879451.642286:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879451.642289:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879451.642291:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.642293:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879451.642297:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.642302:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305062924 00000100:00000001:2.0:1788879451.642306:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305063465 00000100:00000001:2.0:1788879451.642309:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305062924 00000100:00000001:2.0:1788879451.642312:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305063465 00000100:00000001:2.0:1788879451.642315:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879451.642317:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.642320:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879451.642325:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879451/real 1788879451] req@ffff94e50449c700 x1875765290164736/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879451.642336:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879451.642338:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879451.642341:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879451.642343:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879451.642345:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642347:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879451.642349:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879451.642354:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290164736/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879451.642366:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642370:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879451.642375:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642378:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879451.642381:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642384:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879451.642410:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879451.642416:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879451.642423:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879451.642428:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879451.642435:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290164736:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879451.642443:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879451.642446:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879451.642449:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879451.642453:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879451.642457:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879451.642461:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879451.642465:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290164864:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879451.642470:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879451.642472:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879451.642475:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879451.642477:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879451.642481:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.642490:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879451.642492:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:2.0:1788879451.642497:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879451.642515:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879451.642521:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879451.642524:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879451.642526:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.642528:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879451.642532:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.642537:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305062922 00000100:00000001:2.0:1788879451.642540:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305063465 00000100:00000001:2.0:1788879451.642543:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305062922 00000100:00000001:2.0:1788879451.642547:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305063465 00000100:00000001:2.0:1788879451.642550:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879451.642552:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.642555:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879451.642560:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879451/real 1788879451] req@ffff94e50449e300 x1875765290164864/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879451.642569:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879451.642571:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879451.642574:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879451.642576:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879451.642578:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642581:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879451.642582:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879451.642587:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290164864/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879451.642596:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642599:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879451.642602:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642603:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879451.642605:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642607:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879451.642626:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879451.642629:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879451.642634:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879451.642638:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879451.642644:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290164864:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879451.642652:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879451.642654:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879451.642658:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879451.642662:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879451.642665:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879451.642669:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879451.642675:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290164992:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879451.642681:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879451.642684:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879451.642687:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879451.642690:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879451.642696:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.642708:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879451.642711:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:2.0:1788879451.642717:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879451.642742:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879451.642750:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879451.642754:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879451.642757:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.642761:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879451.642766:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.642772:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305062922 00000100:00000001:2.0:1788879451.642809:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305063465 00000100:00000001:2.0:1788879451.642815:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305062922 00000100:00000001:2.0:1788879451.642819:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305063465 00000100:00000001:2.0:1788879451.642824:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879451.642828:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.642832:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879451.642841:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879451/real 1788879451] req@ffff94e50449c000 x1875765290164992/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879451.642859:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879451.642861:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879451.642866:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879451.642869:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879451.642871:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642874:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879451.642876:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879451.642881:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290164992/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879451.642890:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642893:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879451.642896:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642898:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879451.642899:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642930:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879451.642949:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879451.642953:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879451.642957:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879451.642960:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879451.642964:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290164992:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879451.642969:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642972:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879451.642974:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879451.642976:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879451.642980:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879451.642982:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879451.642984:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879451.642987:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879451.642989:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879451.642992:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879451.642993:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.642995:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879451.642997:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879451.642999:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879451.643001:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879451.643002:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879451.643003:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879451.643005:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879451.643007:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879451.643009:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879451.643011:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.643013:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879451.643015:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879451.643016:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879451.643019:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879451.643020:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879451.643022:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879451.643023:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879451.643024:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879451.643026:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879451.643028:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.643030:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879451.643034:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879451.643036:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879451.643039:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879451.643041:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879451.705029:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879451.705035:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879451.705038:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879451.705043:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879451.705047:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879451.705051:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879451.705055:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879451.705058:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879451.705061:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879452.408990:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879452.408996:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879452.409000:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879452.409001:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879452.409021:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879452.409023:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879452.409026:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879452.729027:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879452.729037:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879452.729040:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879452.729047:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879452.729052:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879452.729057:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879452.729061:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879452.729065:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879452.729068:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879453.432986:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879453.432991:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879453.432997:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879453.432998:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879453.433017:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879453.433020:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879453.433023:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879453.753012:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879453.753019:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879453.753023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879453.753029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879453.753033:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879453.753039:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879453.753042:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879453.753045:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879453.753048:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879454.457003:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879454.457008:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879454.457012:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879454.457015:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879454.457017:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879454.457020:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879454.457022:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879454.457024:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879454.457025:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879454.457027:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879454.457046:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879454.457048:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879454.457050:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879454.776962:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879454.776969:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879454.776977:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879454.776982:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879454.776985:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879454.776988:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879454.776991:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879454.776993:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879454.776995:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879455.481037:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879455.481046:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879455.481055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879455.481057:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879455.481084:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879455.481087:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879455.481091:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879455.801051:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879455.801061:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879455.801065:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879455.801073:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879455.801079:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879455.801085:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879455.801089:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879455.801092:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879455.801097:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879456.505050:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879456.505058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879456.505066:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879456.505068:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879456.505099:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879456.505103:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879456.505107:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879456.761034:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879456.761046:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879456.761050:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879456.761079:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879456.761085:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879456.761093:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10395 00000100:00080000:0.0:1788879456.761104:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879456.761107:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879456.761109:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879456.761116:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879456.761118:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879456.761121:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879456.761124:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879456.761125:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879456.761127:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879456.761151:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879456.761164:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879456.761167:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879456.761170:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879456.761173:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879456.761175:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879456.761177:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879456.761188:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879456.761190:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879456.761193:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879456.761196:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879456.761201:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879456.761212:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879456.761221:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290165120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879456.761230:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879456.761233:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879456.761245:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879456.761247:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879456.761250:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879456.761252:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879456.761253:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879456.761256:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879456.761258:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10395 00000100:00080000:0.0:1788879456.761262:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879456.761264:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879456.761266:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879456.761269:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879456.761270:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879456.761272:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879456.761274:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879456.761275:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879456.761276:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879456.761280:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879456.761280:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879456.761285:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1788879456.761286:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879456.761288:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879456.761292:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.761292:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879456.761298:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879456.761299:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879456.761303:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879456.761306:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879456.761310:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1788879456.761312:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290165120:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788879456.761314:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879456.761319:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879456.761322:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:0.0:1788879456.761323:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879456.761328:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879456.761329:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788879456.761330:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879456.761334:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879456.761335:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879456.761339:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879456.761341:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879456.761342:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879456.761345:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879456.761347:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879456.761350:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879456.761352:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879456.761375:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897aa00 x1875765290165248/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879456.761383:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879456.761384:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879456.761387:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879456.761389:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879456.761392:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879456.761393:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879456.761394:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879456.761397:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879456.761398:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879456.761402:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00080000:0.0:1788879456.761403:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10395 00000100:00000001:2.0:1788879456.761408:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788879456.761410:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879456.761413:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879456.761414:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879456.761416:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879456.761417:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879456.761419:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879456.761436:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879456.761438:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879456.761439:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879456.761442:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879456.761445:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879456.761446:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879456.761449:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879456.761451:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879456.761453:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879456.761455:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879456.761457:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879456.761458:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879456.761462:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879456.761465:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788879456.761466:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879456.761467:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879456.761471:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879456.761472:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879456.761476:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879456.761480:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978700 x1875765290165376/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879456.761486:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879456.761488:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.761489:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879456.761490:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879456.761495:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879456.761495:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1788879456.761500:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879456.761500:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.761505:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879456.761506:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879456.761509:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879456.761513:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879456.761514:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879456.761517:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10395 00000100:00000001:2.0:1788879456.761520:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305068585 00000100:00080000:0.0:1788879456.761522:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879456.761526:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879456.761527:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879456.761529:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879456.761530:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879456.761532:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879456.761532:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1788879456.761535:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879456.761538:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879456.761540:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879456.761541:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305068585 00000100:00080000:0.0:1788879456.761544:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879456.761547:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788879456.761549:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879456.761553:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879456.761553:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879456.761558:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879456.761558:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879456.761565:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.761565:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879456.761569:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879456.761571:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879456.761575:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879456.761576:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879456/real 1788879456] req@ffff94e508978a80 x1875765290165120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879456.761577:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879456.761601:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879456.761604:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879456.761606:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879456.761608:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:2.0:1788879456.761608:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1788879456.761610:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879456.761611:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879456.761616:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.761617:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879456.761622:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00100000:0.0:1788879456.761624:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290165504/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879456.761625:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.761630:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879456.761632:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879456.761639:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1788879456.761641:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290165120/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879456.761642:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879456.761647:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.761656:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.761659:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879456.761661:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.761663:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879456.761663:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.761665:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879456.761667:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879456.761669:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879456.761673:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879456.761676:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879456.761679:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290165120:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879456.761682:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.761685:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879456.761686:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879456.761688:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879456.761690:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879456.761691:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879456.761693:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879456.761696:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879456.761698:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879456.761700:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879456.761701:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.761703:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.761709:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879456.761710:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.761712:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879456.761713:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879456.761715:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879456.761716:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879456.761717:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879456.761718:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879456.761720:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879456.761721:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879456.761724:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290165248:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879456.761726:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879456.761728:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879456.761729:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879456.761731:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879456.761733:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.761737:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879456.761738:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:2.0:1788879456.761756:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879456.761768:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879456.761771:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879456.761773:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879456.761774:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.761776:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879456.761778:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.761781:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305068043 00000100:00000001:2.0:1788879456.761783:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305068585 00000100:00000001:2.0:1788879456.761785:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305068043 00000100:00000001:2.0:1788879456.761799:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305068585 00000100:00000001:2.0:1788879456.761801:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879456.761802:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.761804:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879456.761806:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879456/real 1788879456] req@ffff94e50897aa00 x1875765290165248/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879456.761812:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879456.761813:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879456.761815:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879456.761816:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879456.761817:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.761819:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879456.761820:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879456.761839:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290165248/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879456.761844:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.761846:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879456.761848:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.761849:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879456.761850:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.761851:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879456.761866:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879456.761868:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879456.761870:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879456.761871:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879456.761874:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290165248:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879456.761877:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879456.761878:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879456.761879:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879456.761881:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879456.761883:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879456.761884:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879456.761886:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290165376:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879456.761889:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879456.761890:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879456.761891:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879456.761892:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879456.761895:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.761900:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879456.761901:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00000001:2.0:1788879456.761904:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879456.761912:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879456.761915:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879456.761943:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879456.761945:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.761946:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879456.761949:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.761952:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305068042 00000100:00000001:2.0:1788879456.761954:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305068585 00000100:00000001:2.0:1788879456.761956:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305068042 00000100:00000001:2.0:1788879456.761958:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305068585 00000100:00000001:2.0:1788879456.761959:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879456.761961:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.761962:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879456.761965:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879456/real 1788879456] req@ffff94e508978700 x1875765290165376/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879456.761971:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879456.761972:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879456.761974:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879456.761975:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879456.761994:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.761996:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879456.761997:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879456.762000:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290165376/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879456.762005:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.762007:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879456.762008:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.762009:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879456.762010:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.762011:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879456.762025:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879456.762027:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879456.762029:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879456.762030:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879456.762032:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290165376:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879456.762035:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879456.762036:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879456.762037:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879456.762039:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879456.762040:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879456.762042:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879456.762044:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290165504:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879456.762046:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879456.762047:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879456.762048:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879456.762050:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879456.762068:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.762072:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879456.762073:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:2.0:1788879456.762075:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879456.762085:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879456.762088:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879456.762090:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879456.762091:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.762093:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879456.762094:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.762097:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305068042 00000100:00000001:2.0:1788879456.762098:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305068585 00000100:00000001:2.0:1788879456.762100:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305068042 00000100:00000001:2.0:1788879456.762102:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305068585 00000100:00000001:2.0:1788879456.762103:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879456.762105:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.762106:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879456.762109:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879456/real 1788879456] req@ffff94e50897a300 x1875765290165504/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879456.762114:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879456.762115:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879456.762116:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879456.762117:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879456.762118:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.762119:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879456.762120:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879456.762123:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290165504/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879456.762128:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.762129:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879456.762131:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.762132:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879456.762133:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.762134:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879456.762147:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879456.762149:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879456.762151:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879456.762153:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879456.762166:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290165504:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879456.762168:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.762170:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879456.762170:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879456.762172:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879456.762173:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879456.762174:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879456.762175:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879456.762177:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879456.762178:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879456.762180:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879456.762181:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.762182:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879456.762183:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879456.762184:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879456.762185:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879456.762186:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879456.762187:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879456.762188:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879456.762189:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879456.762190:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879456.762191:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.762192:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879456.762193:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879456.762193:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879456.762194:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879456.762195:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879456.762196:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879456.762197:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879456.762198:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879456.762199:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879456.762199:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.762201:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879456.762210:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879456.762211:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879456.762213:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879456.762214:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879456.824977:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879456.824982:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879456.824984:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879456.824996:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879456.824999:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879456.825002:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879456.825005:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879456.825007:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879456.825009:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879457.529025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879457.529032:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879457.529035:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879457.529039:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879457.529044:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879457.529046:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879457.529048:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879457.529050:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879457.529053:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879457.529055:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879457.529078:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879457.529081:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879457.529084:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879457.849008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879457.849015:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879457.849017:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879457.849021:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879457.849024:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879457.849027:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879457.849029:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879457.849038:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879457.849040:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879458.552972:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879458.552977:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879458.552982:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879458.552984:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879458.553001:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879458.553003:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879458.553006:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879458.872979:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879458.872986:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879458.872987:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879458.872991:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879458.872994:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879458.872999:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879458.873002:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879458.873005:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879458.873008:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879459.577004:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879459.577011:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879459.577017:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879459.577018:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879459.577040:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879459.577043:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879459.577047:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879459.897002:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879459.897009:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879459.897012:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879459.897017:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879459.897020:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879459.897024:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879459.897027:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879459.897030:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879459.897032:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879460.600997:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879460.601003:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879460.601008:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879460.601009:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879460.601037:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879460.601039:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879460.601042:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879460.921020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879460.921026:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879460.921028:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879460.921033:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879460.921036:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879460.921040:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879460.921042:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879460.921044:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879460.921046:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879461.625065:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879461.625076:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879461.625085:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879461.625088:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879461.625133:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879461.625138:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879461.625143:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879461.881048:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879461.881064:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879461.881069:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879461.881075:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879461.881083:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879461.881094:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10400 00000100:00080000:1.0:1788879461.881110:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879461.881116:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879461.881120:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879461.881134:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879461.881139:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879461.881144:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879461.881149:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879461.881152:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879461.881155:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879461.881163:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879461.881167:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879461.881172:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879461.881177:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879461.881182:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879461.881186:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879461.881189:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879461.881204:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879461.881208:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879461.881212:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879461.881216:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879461.881222:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879461.881237:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879461.881249:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100bb80 x1875765290165632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879461.881263:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879461.881269:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879461.881289:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879461.881294:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879461.881300:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879461.881303:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879461.881308:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879461.881314:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879461.881321:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10400 00000100:00000001:2.0:1788879461.881325:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1788879461.881330:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879461.881333:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879461.881339:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879461.881340:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879461.881347:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879461.881349:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879461.881352:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879461.881365:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1788879461.881366:0:80852:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1788879461.881371:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1788879461.881373:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:2.0:1788879461.881378:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879461.881382:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00100000:2.0:1788879461.881390:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290165632:192.168.203.129@tcp:250:kworker.0 00000020:00000001:1.0:1788879461.881393:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879461.881401:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788879461.881406:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1788879461.881409:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1788879461.881413:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879461.881415:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879461.881421:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1788879461.881425:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879461.881432:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879461.881432:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879461.881443:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879461.881452:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879461.881456:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879461.881459:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879461.881461:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000008:00100000:1.0:1788879461.881465:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879461.881467:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286523264 : -117763423028352 : ffff94e51100bb80) 00000008:00000001:1.0:1788879461.881476:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.881479:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879461.881482:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879461.881492:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879461.881494:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879461.881497:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879461.881500:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879461.881503:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879461.881512:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879461.881521:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008700 x1875765290165760/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1788879461.881536:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879461.881537:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879461.881545:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879461.881552:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879461.881556:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879461.881557:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:1.0:1788879461.881565:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879461.881567:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879461.881569:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879461.881574:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879461.881575:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879461.881584:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879461.881586:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879461.881595:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879461.881596:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10400 00000100:00000001:2.0:1788879461.881608:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305073705 00000100:00080000:1.0:1788879461.881610:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879461.881617:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879461.881620:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879461.881626:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:1.0:1788879461.881627:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879461.881633:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879461.881637:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879461.881640:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879461.881643:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879461.881645:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879461.881651:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879461.881652:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305073705 00000100:00000001:1.0:1788879461.881660:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.881663:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1788879461.881671:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879461.881673:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879461.881682:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879461.881683:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879461.881691:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1788879461.881693:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879461.881701:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:2.0:1788879461.881731:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879461/real 1788879461] req@ffff94e51100bb80 x1875765290165632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1788879461.881735:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879461.881750:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879461.881762:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879461.881778:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000400:00000001:2.0:1788879461.881780:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1788879461.881782:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879461.881789:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879461.881790:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879461.881796:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879461.881801:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.881805:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879461.881810:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.881814:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879461.881823:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879461.881824:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008e00 x1875765290165888/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879461.881832:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879461.881839:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879461.881850:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879461.881858:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879461.881858:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100bb80 x1875765290165632/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879461.881867:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879461.881877:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879461.881889:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788879461.881890:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879461.881898:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879461.881899:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879461.881945:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.881948:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879461.881950:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.881953:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879461.881957:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879461.881958:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879461.881964:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1788879461.881969:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879461.881975:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879461.881983:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:1.0:1788879461.881984:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10400 00000100:00100000:2.0:1788879461.881995:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290165632:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1788879461.882001:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879461.882004:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882012:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879461.882014:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879461.882017:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1788879461.882019:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879461.882024:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:1.0:1788879461.882027:0:80852:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1788879461.882031:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879461.882032:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879461.882038:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879461.882040:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879461.882042:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000020:00000001:1.0:1788879461.882048:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:2.0:1788879461.882052:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1788879461.882054:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1788879461.882058:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879461.882060:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879461.882065:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00080000:1.0:1788879461.882072:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879461.882072:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882079:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879461.882081:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879461.882087:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879461.882091:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879461.882093:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:1.0:1788879461.882095:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882099:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879461.882101:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879461.882106:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:1.0:1788879461.882108:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879461.882113:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00000001:1.0:1788879461.882115:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882118:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879461.882121:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882125:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879461.882130:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879461.882131:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879461.882135:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879461.882136:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879461.882142:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879461.882143:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879461.882149:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879461.882152:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879461.882155:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879461.882164:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290165760:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879461.882167:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882179:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1788879461.882183:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008000 x1875765290166016/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788879461.882187:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879461.882195:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879461.882203:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788879461.882206:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788879461.882210:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879461.882218:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879461.882219:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882236:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879461.882238:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286509824 : -117763423041792 : ffff94e511008700) 00000100:00000001:2.0:1788879461.882245:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879461.882273:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879461.882281:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879461.882285:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879461.882287:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882289:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879461.882293:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882298:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305073164 00000100:00000001:2.0:1788879461.882302:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305073705 00000100:00000001:2.0:1788879461.882305:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305073164 00000100:00000001:2.0:1788879461.882308:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305073705 00000100:00000001:2.0:1788879461.882310:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879461.882313:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882315:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879461.882321:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879461/real 1788879461] req@ffff94e511008700 x1875765290165760/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879461.882332:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879461.882333:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879461.882336:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879461.882338:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879461.882341:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882343:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879461.882345:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879461.882350:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008700 x1875765290165760/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879461.882358:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882361:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879461.882364:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882366:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879461.882367:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882369:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879461.882387:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879461.882390:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879461.882394:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879461.882397:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879461.882400:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290165760:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879461.882405:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879461.882407:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879461.882408:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879461.882411:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879461.882414:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879461.882416:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879461.882420:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290165888:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879461.882424:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879461.882427:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879461.882429:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879461.882431:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879461.882434:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882440:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879461.882441:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286511616 : -117763423040000 : ffff94e511008e00) 00000100:00000001:2.0:1788879461.882445:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879461.882458:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879461.882462:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879461.882465:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879461.882467:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882469:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879461.882472:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882475:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305073162 00000100:00000001:2.0:1788879461.882478:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305073705 00000100:00000001:2.0:1788879461.882481:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305073162 00000100:00000001:2.0:1788879461.882484:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305073705 00000100:00000001:2.0:1788879461.882486:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879461.882489:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882491:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879461.882495:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879461/real 1788879461] req@ffff94e511008e00 x1875765290165888/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879461.882504:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879461.882506:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879461.882508:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879461.882510:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879461.882512:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882514:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879461.882516:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879461.882520:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008e00 x1875765290165888/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879461.882528:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882531:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879461.882533:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882535:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879461.882537:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882539:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879461.882557:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879461.882563:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879461.882569:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879461.882574:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879461.882580:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290165888:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879461.882587:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882590:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879461.882593:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879461.882596:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879461.882600:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879461.882603:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879461.882605:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879461.882610:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879461.882613:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879461.882616:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879461.882619:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882623:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879461.882625:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879461.882627:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879461.882631:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879461.882633:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879461.882635:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879461.882639:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879461.882641:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879461.882645:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879461.882647:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882651:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882658:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879461.882660:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882664:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879461.882667:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879461.882670:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879461.882672:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879461.882675:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879461.882680:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879461.882683:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879461.882687:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879461.882694:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290166016:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879461.882702:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879461.882729:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879461.882732:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879461.882737:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879461.882743:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882760:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879461.882763:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508032 : -117763423043584 : ffff94e511008000) 00000100:00000001:2.0:1788879461.882770:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879461.882797:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879461.882804:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879461.882807:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879461.882809:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882811:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879461.882815:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882820:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305073162 00000100:00000001:2.0:1788879461.882823:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305073705 00000100:00000001:2.0:1788879461.882826:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305073162 00000100:00000001:2.0:1788879461.882829:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305073705 00000100:00000001:2.0:1788879461.882832:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879461.882834:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882837:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879461.882842:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879461/real 1788879461] req@ffff94e511008000 x1875765290166016/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879461.882852:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879461.882853:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879461.882856:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879461.882858:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879461.882860:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882862:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879461.882864:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879461.882869:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008000 x1875765290166016/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879461.882877:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882880:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879461.882883:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882886:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879461.882887:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882890:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879461.882935:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879461.882939:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879461.882944:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879461.882947:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879461.882951:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290166016:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879461.882955:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882958:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879461.882959:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879461.882961:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879461.882965:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879461.882966:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879461.882968:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879461.882971:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879461.882973:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879461.882975:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879461.882977:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882979:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879461.882983:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879461.882985:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879461.882987:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879461.882989:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879461.945029:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879461.945037:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879461.945039:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879461.945047:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879461.945051:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879461.945056:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879461.945059:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879461.945062:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879461.945065:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879462.649007:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879462.649015:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879462.649022:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879462.649025:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879462.649057:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879462.649063:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879462.649068:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879462.969058:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879462.969069:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879462.969073:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879462.969082:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879462.969089:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879462.969095:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879462.969101:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879462.969106:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879462.969111:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879463.673029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879463.673036:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879463.673042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879463.673043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879463.673071:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879463.673076:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879463.673081:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879463.993037:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879463.993045:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879463.993048:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879463.993054:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879463.993059:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879463.993064:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879463.993067:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879463.993072:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879463.993077:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879464.696984:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879464.696989:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879464.696994:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879464.696997:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879464.696999:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879464.697002:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879464.697005:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879464.697007:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879464.697008:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879464.697010:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879464.697029:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879464.697031:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879464.697033:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879465.017021:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879465.017028:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879465.017031:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879465.017036:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879465.017040:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879465.017044:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879465.017047:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879465.017050:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879465.017052:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879465.721002:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879465.721010:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879465.721018:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879465.721020:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879465.721051:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879465.721055:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879465.721060:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879466.041029:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879466.041038:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879466.041041:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879466.041048:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879466.041053:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879466.041058:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879466.041062:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879466.041066:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879466.041070:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879466.745023:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879466.745031:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879466.745037:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879466.745039:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879466.745108:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879466.745111:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879466.745115:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879467.001122:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879467.001134:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879467.001138:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879467.001143:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879467.001148:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879467.001156:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10405 00000100:00080000:0.0:1788879467.001169:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879467.001174:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879467.001176:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879467.001185:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879467.001188:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879467.001192:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879467.001195:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879467.001197:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879467.001199:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879467.001203:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879467.001206:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879467.001210:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879467.001214:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879467.001218:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879467.001220:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879467.001223:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879467.001237:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879467.001239:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879467.001244:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879467.001248:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879467.001255:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879467.001273:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879467.001287:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290166144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879467.001306:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879467.001313:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879467.001337:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879467.001342:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879467.001348:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879467.001351:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879467.001355:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879467.001359:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879467.001364:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10405 00000100:00080000:0.0:1788879467.001371:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879467.001374:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879467.001377:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879467.001378:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788879467.001383:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879467.001385:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788879467.001388:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879467.001391:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788879467.001393:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879467.001396:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1788879467.001399:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879467.001404:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879467.001405:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879467.001409:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879467.001413:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.001413:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879467.001416:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879467.001422:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879467.001425:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879467.001428:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879467.001434:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879467.001436:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879467.001440:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879467.001447:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879467.001448:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290166144:192.168.203.129@tcp:250:kworker.0 00000008:00000001:0.0:1788879467.001456:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.001459:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788879467.001462:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879467.001466:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879467.001471:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879467.001472:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879467.001478:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879467.001479:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879467.001484:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879467.001486:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879467.001488:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879467.001490:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879467.001503:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.001514:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00100000:0.0:1788879467.001515:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290166272/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879467.001520:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:2.0:1788879467.001529:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879467.001541:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879467.001547:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879467.001554:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879467.001559:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879467.001566:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879467.001570:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879467.001573:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1788879467.001581:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879467.001583:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879467.001592:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10405 00000100:00000001:2.0:1788879467.001599:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:0.0:1788879467.001603:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879467.001607:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879467.001609:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879467.001612:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879467.001613:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879467.001617:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879467.001625:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.001630:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305078825 00000020:00000001:0.0:1788879467.001634:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879467.001638:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879467.001641:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879467.001643:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1788879467.001645:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879467.001651:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879467.001653:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879467.001654:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305078825 00000100:00080000:0.0:1788879467.001662:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879467.001663:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788879467.001690:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.001695:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879467.001696:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879467.001703:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879467.001707:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879467.001708:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1788879467.001717:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879467.001729:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879467.001735:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879467.001735:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879467/real 1788879467] req@ffff94e508979c00 x1875765290166144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1788879467.001746:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879467.001755:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879467.001763:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1788879467.001768:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000400:00000001:2.0:1788879467.001769:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1788879467.001774:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.001779:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879467.001780:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879467.001785:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879467.001788:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:0.0:1788879467.001789:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.001791:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.001797:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879467.001800:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879467.001801:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879467.001809:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290166144/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879467.001818:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290166400/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879467.001833:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.001841:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879467.001843:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879467.001850:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879467.001851:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.001859:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879467.001862:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.001863:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879467.001868:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879467.001869:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879467.001873:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879467.001878:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:2.0:1788879467.001881:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1788879467.001883:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879467.001888:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1788879467.001890:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879467.001895:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879467.001897:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879467.001941:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290166144:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1788879467.001953:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10405 00000100:00000001:2.0:1788879467.001957:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879467.001961:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879467.001963:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879467.001967:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1788879467.001969:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1788879467.001972:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1788879467.001973:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879467.001989:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879467.001991:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879467.001994:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:0.0:1788879467.001996:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879467.001998:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000020:00000001:0.0:1788879467.002002:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:2.0:1788879467.002006:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:0.0:1788879467.002007:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:2.0:1788879467.002011:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879467.002013:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879467.002017:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879467.002018:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879467.002022:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879467.002024:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879467.002026:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879467.002032:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879467.002036:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879467.002039:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1788879467.002040:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879467.002046:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002056:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879467.002057:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879467.002062:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879467.002064:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:0.0:1788879467.002070:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002070:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879467.002074:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879467.002076:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002079:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879467.002083:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879467.002084:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879467.002086:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1788879467.002089:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879467.002091:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879467.002093:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879467.002096:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879467.002098:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879467.002103:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879467.002104:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290166272:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788879467.002112:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002112:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879467.002118:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879467.002122:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879467.002125:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290166528/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788879467.002128:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879467.002138:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879467.002140:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879467.002146:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879467.002151:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002156:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879467.002161:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:2.0:1788879467.002170:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879467.002198:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879467.002207:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879467.002212:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879467.002214:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002217:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879467.002222:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002228:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305078284 00000100:00000001:2.0:1788879467.002231:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305078825 00000100:00000001:2.0:1788879467.002234:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305078284 00000100:00000001:2.0:1788879467.002237:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305078825 00000100:00000001:2.0:1788879467.002239:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879467.002241:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002243:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879467.002249:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879467/real 1788879467] req@ffff94e50897ad80 x1875765290166272/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879467.002259:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879467.002261:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879467.002264:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879467.002266:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879467.002267:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002269:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879467.002270:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879467.002275:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290166272/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879467.002282:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002285:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879467.002287:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002289:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879467.002291:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002293:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879467.002310:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879467.002313:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879467.002317:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879467.002320:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879467.002323:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290166272:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879467.002327:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879467.002328:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879467.002330:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879467.002332:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879467.002334:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879467.002347:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879467.002350:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290166400:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879467.002354:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879467.002356:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879467.002358:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879467.002360:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879467.002364:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002369:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879467.002371:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:2.0:1788879467.002375:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879467.002389:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879467.002393:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879467.002395:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879467.002397:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002399:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879467.002402:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002405:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305078282 00000100:00000001:2.0:1788879467.002408:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305078825 00000100:00000001:2.0:1788879467.002410:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305078282 00000100:00000001:2.0:1788879467.002413:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305078825 00000100:00000001:2.0:1788879467.002415:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879467.002417:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002419:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879467.002423:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879467/real 1788879467] req@ffff94e508978e00 x1875765290166400/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879467.002432:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879467.002433:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879467.002435:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879467.002437:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879467.002439:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002441:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879467.002442:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879467.002446:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290166400/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879467.002453:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002455:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879467.002457:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002459:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879467.002461:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002463:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879467.002479:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879467.002481:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879467.002484:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879467.002487:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879467.002491:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290166400:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879467.002494:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002496:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879467.002498:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879467.002499:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879467.002501:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879467.002503:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879467.002505:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879467.002507:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879467.002509:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879467.002513:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879467.002514:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002516:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879467.002517:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879467.002519:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879467.002520:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879467.002522:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879467.002523:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879467.002525:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879467.002526:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879467.002528:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879467.002529:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002531:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002535:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879467.002537:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002539:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879467.002540:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879467.002541:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879467.002543:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879467.002544:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879467.002546:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879467.002548:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879467.002550:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879467.002554:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290166528:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879467.002557:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879467.002559:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879467.002561:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879467.002562:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879467.002565:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002569:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879467.002570:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:2.0:1788879467.002574:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879467.002584:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879467.002587:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879467.002589:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879467.002591:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002593:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879467.002596:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002599:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305078282 00000100:00000001:2.0:1788879467.002602:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305078825 00000100:00000001:2.0:1788879467.002604:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305078282 00000100:00000001:2.0:1788879467.002607:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305078825 00000100:00000001:2.0:1788879467.002609:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879467.002611:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002613:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879467.002617:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879467/real 1788879467] req@ffff94e50897bb80 x1875765290166528/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879467.002624:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879467.002626:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879467.002628:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879467.002629:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879467.002631:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002632:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879467.002634:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879467.002637:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290166528/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879467.002644:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002646:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879467.002649:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002651:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879467.002652:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002654:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879467.002693:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879467.002696:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879467.002700:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879467.002703:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879467.002707:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290166528:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879467.002728:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002731:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879467.002733:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879467.002735:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879467.002738:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879467.002739:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879467.002741:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879467.002744:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879467.002746:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879467.002749:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879467.002750:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002752:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879467.002756:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879467.002758:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879467.002771:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879467.002773:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879467.065005:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879467.065011:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879467.065013:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879467.065018:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879467.065022:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879467.065025:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879467.065028:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879467.065030:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879467.065033:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879467.769007:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879467.769015:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879467.769018:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879467.769022:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879467.769027:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879467.769030:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879467.769032:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879467.769035:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879467.769038:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879467.769040:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879467.769073:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879467.769076:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879467.769079:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879468.089033:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879468.089041:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879468.089044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879468.089051:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879468.089056:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879468.089062:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879468.089066:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879468.089069:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879468.089073:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879468.793112:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879468.793122:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879468.793148:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879468.793150:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879468.793254:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879468.793260:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879468.793283:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879469.113026:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879469.113037:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879469.113041:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879469.113050:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879469.113056:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879469.113062:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879469.113067:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879469.113071:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879469.113075:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879469.817017:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879469.817023:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879469.817029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879469.817030:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879469.817061:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879469.817063:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879469.817067:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879470.136982:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879470.136989:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879470.136992:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879470.137014:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879470.137018:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879470.137022:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879470.137025:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879470.137029:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879470.137031:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879470.840993:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879470.840998:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879470.841002:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879470.841003:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879470.841027:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879470.841030:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879470.841032:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879471.160982:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879471.160991:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879471.160993:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879471.160999:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879471.161003:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879471.161007:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879471.161010:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879471.161013:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879471.161016:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879471.865092:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879471.865100:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879471.865107:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879471.865109:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879471.865147:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879471.865151:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879471.865154:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879472.121102:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879472.121112:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879472.121115:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879472.121119:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879472.121125:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879472.121132:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10410 00000100:00080000:3.0:1788879472.121154:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879472.121157:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879472.121160:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879472.121168:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879472.121171:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879472.121174:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879472.121177:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879472.121179:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879472.121180:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879472.121185:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879472.121187:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879472.121190:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879472.121194:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879472.121198:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879472.121200:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879472.121202:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879472.121214:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879472.121215:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879472.121219:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879472.121240:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879472.121247:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879472.121262:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879472.121275:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290166656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879472.121290:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879472.121306:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879472.121331:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879472.121336:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879472.121341:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879472.121343:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879472.121346:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879472.121350:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879472.121353:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10410 00000100:00080000:3.0:1788879472.121358:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879472.121370:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879472.121373:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879472.121377:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879472.121380:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879472.121383:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879472.121395:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879472.121399:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879472.121401:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879472.121404:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1788879472.121407:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879472.121415:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.121417:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788879472.121418:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879472.121427:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788879472.121431:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879472.121436:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788879472.121437:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879472.121445:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879472.121446:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788879472.121450:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879472.121457:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879472.121460:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879472.121461:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879472.121486:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879472.121488:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879472.121492:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879472.121495:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879472.121496:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879472.121499:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879472.121504:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879472.121510:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290166656:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879472.121512:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.121520:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1788879472.121526:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290166784/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788879472.121528:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879472.121532:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879472.121538:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879472.121544:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879472.121546:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879472.121549:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879472.121556:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879472.121556:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879472.121563:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879472.121585:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879472.121586:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879472.121591:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879472.121595:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10410 00000100:00080000:3.0:1788879472.121601:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879472.121602:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879472.121604:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879472.121612:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:3.0:1788879472.121613:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879472.121625:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879472.121630:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1788879472.121631:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879472.121670:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879472.121676:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879472.121679:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879472.121682:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879472.121687:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879472.121690:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879472.121694:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879472.121698:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879472.121701:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879472.121703:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879472.121707:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879472.121719:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879472.121722:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879472.121730:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879472.121746:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879472.121748:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879472.121750:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879472.121753:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788879472.121755:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879472.121762:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879472.121774:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290166912/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879472.121777:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879472.121786:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879472.121792:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.121801:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879472.121810:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879472.121812:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879472.121816:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.121821:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305083945 00000100:00000001:3.0:1788879472.121824:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879472.121833:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879472.121837:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879472.121839:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879472.121841:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788879472.121842:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879472.121853:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879472.121858:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305083945 00000100:00080000:3.0:1788879472.121859:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10410 00000100:00000001:2.0:1788879472.121870:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788879472.121872:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879472.121879:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879472.121880:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879472.121887:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879472.121889:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879472.121894:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879472.121896:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1788879472.121899:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879472.121957:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:2.0:1788879472.121989:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879472/real 1788879472] req@ffff94e50449f100 x1875765290166656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788879472.121991:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879472.121998:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879472.122002:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879472.122013:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879472.122019:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879472.122021:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:2.0:1788879472.122022:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1788879472.122024:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:2.0:1788879472.122028:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788879472.122029:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879472.122032:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879472.122034:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879472.122037:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879472.122041:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.122045:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:3.0:1788879472.122046:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879472.122050:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.122051:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879472.122055:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879472.122074:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879472.122079:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788879472.122080:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879472.122096:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879472.122097:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879472.122109:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879472.122112:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290166656/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879472.122123:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290167040/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879472.122138:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.122157:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879472.122159:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879472.122164:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879472.122166:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.122170:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879472.122174:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.122175:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.122183:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879472.122186:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879472.122191:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879472.122198:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879472.122202:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879472.122211:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290166656:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879472.122221:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.122226:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879472.122229:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879472.122234:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879472.122240:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879472.122244:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879472.122248:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879472.122253:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879472.122256:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879472.122261:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879472.122264:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.122269:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.122280:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879472.122284:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.122289:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879472.122292:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879472.122295:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879472.122309:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879472.122312:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879472.122317:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879472.122320:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879472.122323:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879472.122327:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290166784:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879472.122332:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879472.122335:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879472.122337:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879472.122340:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879472.122344:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.122354:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879472.122355:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:2.0:1788879472.122372:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879472.122392:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879472.122399:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879472.122403:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879472.122405:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.122407:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879472.122412:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.122419:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305083404 00000100:00000001:2.0:1788879472.122425:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305083945 00000100:00000001:2.0:1788879472.122430:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305083404 00000100:00000001:2.0:1788879472.122434:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305083945 00000100:00000001:2.0:1788879472.122438:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879472.122442:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.122446:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879472.122455:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879472/real 1788879472] req@ffff94e50449ca80 x1875765290166784/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879472.122484:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879472.122487:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879472.122493:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879472.122512:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879472.122514:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.122516:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879472.122518:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879472.122524:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290166784/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879472.122544:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.122547:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879472.122549:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.122550:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879472.122552:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.122554:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879472.122589:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879472.122593:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879472.122599:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879472.122604:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879472.122612:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290166784:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879472.122620:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879472.122623:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879472.122625:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879472.122630:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879472.122656:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879472.122668:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879472.122674:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290166912:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879472.122681:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879472.122685:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879472.122688:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879472.122692:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879472.122698:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.122710:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879472.122714:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:2.0:1788879472.122721:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879472.122748:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879472.122755:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879472.122758:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879472.122760:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.122762:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879472.122766:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.122770:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305083402 00000100:00000001:2.0:1788879472.122774:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305083945 00000100:00000001:2.0:1788879472.122776:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305083402 00000100:00000001:2.0:1788879472.122779:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305083945 00000100:00000001:2.0:1788879472.122782:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879472.122785:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.122787:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879472.122792:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879472/real 1788879472] req@ffff94e50449c000 x1875765290166912/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879472.122811:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879472.122812:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879472.122816:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879472.122818:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879472.122819:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.122821:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879472.122824:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879472.122828:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290166912/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879472.122837:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.122840:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879472.122843:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.122845:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879472.122846:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.122848:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879472.122866:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879472.122869:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879472.122872:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879472.122875:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879472.122879:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290166912:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879472.122883:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879472.122884:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879472.122886:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879472.122888:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879472.122890:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879472.122893:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879472.122896:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290167040:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879472.122901:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879472.122932:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879472.122935:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879472.122937:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879472.122941:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.122947:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879472.122949:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:2.0:1788879472.122953:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879472.122983:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879472.122987:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879472.122989:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879472.122991:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.122992:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879472.122994:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.122997:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305083403 00000100:00000001:2.0:1788879472.123000:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305083945 00000100:00000001:2.0:1788879472.123002:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305083403 00000100:00000001:2.0:1788879472.123004:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305083945 00000100:00000001:2.0:1788879472.123006:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879472.123008:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.123009:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879472.123013:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879472/real 1788879472] req@ffff94e50449e300 x1875765290167040/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879472.123019:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879472.123020:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879472.123022:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879472.123024:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879472.123025:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.123027:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879472.123028:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879472.123031:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290167040/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879472.123037:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.123039:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879472.123041:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.123043:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879472.123044:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.123046:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879472.123060:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879472.123063:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879472.123066:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879472.123068:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879472.123081:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290167040:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879472.123084:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.123086:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879472.123087:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879472.123088:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879472.123090:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879472.123091:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879472.123093:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879472.123095:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879472.123096:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879472.123098:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879472.123099:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.123101:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879472.123102:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879472.123103:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879472.123104:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879472.123105:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879472.123106:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879472.123108:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879472.123109:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879472.123110:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879472.123111:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.123112:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879472.123113:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879472.123114:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879472.123116:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879472.123117:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879472.123118:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879472.123119:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879472.123121:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879472.123122:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879472.123123:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.123125:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879472.123128:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879472.123130:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879472.123132:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879472.123133:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879472.185008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879472.185017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879472.185022:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879472.185030:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879472.185036:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879472.185041:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879472.185045:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879472.185048:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879472.185051:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879472.889034:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879472.889044:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879472.889054:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879472.889056:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879472.889136:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879472.889139:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879472.889143:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879473.208974:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879473.208982:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879473.208984:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879473.208989:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879473.208993:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879473.208996:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879473.208999:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879473.209001:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879473.209003:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879473.913080:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879473.913091:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879473.913108:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879473.913109:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879473.913154:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879473.913158:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879473.913162:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879474.233007:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879474.233017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879474.233020:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879474.233027:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879474.233031:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879474.233036:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879474.233040:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879474.233043:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879474.233047:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879474.937015:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879474.937022:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879474.937031:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879474.937035:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879474.937039:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879474.937043:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879474.937048:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879474.937051:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879474.937054:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879474.937058:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879474.937115:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879474.937118:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879474.937122:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879475.257020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879475.257029:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879475.257032:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879475.257039:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879475.257044:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879475.257048:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879475.257052:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879475.257055:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879475.257059:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879475.961096:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879475.961106:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879475.961116:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879475.961119:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879475.961233:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879475.961237:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879475.961242:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879476.280989:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879476.280996:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879476.281000:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879476.281006:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879476.281010:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879476.281015:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879476.281018:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879476.281021:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879476.281024:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879476.985064:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879476.985071:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879476.985077:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879476.985079:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879476.985137:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879476.985140:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879476.985143:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879477.241040:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879477.241050:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879477.241053:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879477.241057:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879477.241062:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879477.241068:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10416 00000100:00080000:0.0:1788879477.241079:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879477.241083:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879477.241085:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879477.241093:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879477.241095:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879477.241099:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879477.241101:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879477.241103:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879477.241105:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879477.241109:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879477.241112:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879477.241115:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879477.241118:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879477.241121:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879477.241124:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879477.241127:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879477.241138:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879477.241140:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879477.241143:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879477.241146:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879477.241151:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879477.241163:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879477.241172:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290167168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879477.241191:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879477.241196:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879477.241211:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879477.241215:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879477.241219:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879477.241220:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879477.241222:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879477.241233:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879477.241238:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10416 00000100:00080000:0.0:1788879477.241244:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879477.241247:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879477.241250:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879477.241255:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879477.241257:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879477.241260:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788879477.241261:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879477.241267:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788879477.241269:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879477.241272:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879477.241277:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879477.241277:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879477.241282:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879477.241284:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879477.241288:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879477.241293:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1788879477.241294:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:2.0:1788879477.241296:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879477.241305:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879477.241307:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290167168:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788879477.241309:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879477.241316:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1788879477.241319:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879477.241321:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1788879477.241326:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879477.241337:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879477.241338:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:2.0:1788879477.241343:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:0.0:1788879477.241345:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879477.241350:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879477.241350:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879477.241358:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879477.241364:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879477.241366:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879477.241368:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879477.241369:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879477.241371:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.241374:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879477.241378:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000100:00000001:0.0:1788879477.241379:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.241382:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1788879477.241391:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979180 x1875765290167296/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879477.241401:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879477.241404:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879477.241408:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879477.241411:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879477.241415:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879477.241416:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879477.241418:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879477.241421:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879477.241424:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10416 00000800:00000100:2.0:1788879477.241430:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1788879477.241432:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879477.241436:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879477.241439:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879477.241443:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:0.0:1788879477.241444:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879477.241448:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879477.241450:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1788879477.241452:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879477.241465:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879477.241467:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879477.241469:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879477.241474:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879477.241477:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879477.241478:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.241487:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305089065 00000100:00080000:0.0:1788879477.241488:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879477.241494:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879477.241495:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879477.241498:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879477.241500:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1788879477.241501:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879477.241507:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879477.241510:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879477.241512:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305089065 00000008:00000001:0.0:1788879477.241513:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879477.241519:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.241521:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1788879477.241526:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879477.241527:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879477.241529:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.241533:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879477.241534:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879477.241539:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879477.241539:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1788879477.241541:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879477.241549:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879477.241549:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879477/real 1788879477] req@ffff94e50897b800 x1875765290167168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879477.241558:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290167424/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1788879477.241571:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788879477.241575:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:2.0:1788879477.241576:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879477.241580:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.241582:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879477.241584:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.241587:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00080000:0.0:1788879477.241589:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879477.241590:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879477.241621:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879477.241622:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879477.241627:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879477.241655:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879477.241657:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879477.241663:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00100000:2.0:1788879477.241666:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290167168/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879477.241670:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10416 00000100:00080000:0.0:1788879477.241679:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879477.241687:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879477.241688:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879477.241692:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879477.241695:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1788879477.241698:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879477.241701:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788879477.241702:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879477.241704:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1788879477.241707:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879477.241708:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.241725:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000020:00000001:0.0:1788879477.241728:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879477.241730:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879477.241736:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00080000:2.0:1788879477.241738:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1788879477.241741:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879477.241748:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00080000:0.0:1788879477.241749:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879477.241753:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1788879477.241755:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879477.241759:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1788879477.241763:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290167168:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788879477.241765:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879477.241773:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.241776:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879477.241777:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879477.241782:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879477.241782:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879477.241786:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000008:00000001:0.0:1788879477.241788:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879477.241790:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1788879477.241792:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879477.241796:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879477.241800:0:2361:0:(genops.c:1216:class_import_put()) Process entered 02000000:00000001:0.0:1788879477.241801:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000020:00000001:2.0:1788879477.241803:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:0.0:1788879477.241805:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879477.241809:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1788879477.241810:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879477.241816:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879477.241816:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:0.0:1788879477.241821:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.241823:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879477.241827:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879477.241830:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.241831:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879477.241843:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290167552/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879477.241845:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879477.241851:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879477.241855:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879477.241858:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879477.241859:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.241865:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879477.241867:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.241893:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879477.241898:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879477.241900:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879477.241916:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879477.241919:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879477.241923:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879477.241929:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290167296:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879477.241936:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879477.241979:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879477.241983:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879477.241986:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879477.242001:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.242010:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879477.242012:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 00000100:00000001:2.0:1788879477.242018:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879477.242039:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879477.242046:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879477.242050:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879477.242052:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.242054:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879477.242058:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.242064:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305088524 00000100:00000001:2.0:1788879477.242068:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305089065 00000100:00000001:2.0:1788879477.242071:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305088524 00000100:00000001:2.0:1788879477.242074:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305089065 00000100:00000001:2.0:1788879477.242077:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879477.242080:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.242083:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879477.242090:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879477/real 1788879477] req@ffff94e508979180 x1875765290167296/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879477.242102:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879477.242104:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879477.242107:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879477.242109:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879477.242110:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242112:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879477.242114:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879477.242117:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290167296/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879477.242124:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242126:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879477.242129:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242130:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879477.242131:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242133:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879477.242149:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879477.242151:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879477.242163:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879477.242165:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879477.242168:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290167296:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879477.242172:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879477.242173:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879477.242175:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879477.242177:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879477.242178:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879477.242180:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879477.242183:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290167424:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879477.242188:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879477.242190:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879477.242192:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879477.242193:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879477.242196:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.242202:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879477.242203:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 00000100:00000001:2.0:1788879477.242207:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879477.242219:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879477.242223:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879477.242226:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879477.242227:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.242228:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879477.242231:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.242234:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305088522 00000100:00000001:2.0:1788879477.242236:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305089065 00000100:00000001:2.0:1788879477.242238:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305088522 00000100:00000001:2.0:1788879477.242241:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305089065 00000100:00000001:2.0:1788879477.242243:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879477.242244:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.242246:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879477.242249:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879477/real 1788879477] req@ffff94e508978380 x1875765290167424/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879477.242256:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879477.242257:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879477.242259:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879477.242260:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879477.242262:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242263:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879477.242265:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879477.242268:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290167424/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879477.242280:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242282:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879477.242284:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242285:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879477.242286:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242288:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879477.242301:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879477.242303:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879477.242306:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879477.242308:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879477.242310:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290167424:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879477.242314:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242315:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879477.242317:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879477.242318:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879477.242320:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879477.242321:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879477.242323:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879477.242325:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879477.242326:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879477.242329:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879477.242330:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242332:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879477.242333:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879477.242334:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879477.242336:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879477.242337:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879477.242338:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879477.242340:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879477.242341:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879477.242342:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879477.242343:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242345:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242348:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879477.242350:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.242352:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879477.242354:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879477.242355:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879477.242356:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879477.242358:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879477.242359:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879477.242360:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879477.242362:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879477.242365:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290167552:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879477.242367:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879477.242369:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879477.242370:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879477.242372:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879477.242374:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.242378:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879477.242379:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:2.0:1788879477.242383:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879477.242391:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879477.242395:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879477.242397:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879477.242398:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.242399:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879477.242401:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.242404:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305088522 00000100:00000001:2.0:1788879477.242407:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305089065 00000100:00000001:2.0:1788879477.242409:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305088522 00000100:00000001:2.0:1788879477.242411:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305089065 00000100:00000001:2.0:1788879477.242413:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879477.242414:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.242416:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879477.242419:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879477/real 1788879477] req@ffff94e50897bb80 x1875765290167552/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879477.242431:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879477.242433:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879477.242435:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879477.242436:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879477.242437:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242439:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879477.242440:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879477.242444:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290167552/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879477.242458:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242460:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879477.242462:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242463:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879477.242464:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242466:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879477.242479:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879477.242481:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879477.242484:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879477.242485:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879477.242488:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290167552:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879477.242491:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242492:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879477.242493:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879477.242494:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879477.242496:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879477.242497:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879477.242498:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879477.242500:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879477.242501:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879477.242503:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879477.242504:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242507:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879477.242509:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879477.242510:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879477.242512:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879477.242513:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879477.305058:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879477.305067:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879477.305070:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879477.305077:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879477.305082:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879477.305087:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879477.305091:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879477.305095:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879477.305099:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879478.009004:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879478.009014:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879478.009017:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879478.009022:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879478.009029:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879478.009032:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879478.009036:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879478.009040:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879478.009046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879478.009048:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879478.009076:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879478.009079:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879478.009082:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879478.329005:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879478.329014:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879478.329017:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879478.329040:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879478.329044:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879478.329049:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879478.329053:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879478.329056:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879478.329058:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879479.033065:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879479.033074:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879479.033100:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879479.033103:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879479.033176:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879479.033181:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879479.033188:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879479.353013:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879479.353022:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879479.353026:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879479.353032:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879479.353037:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879479.353042:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879479.353046:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879479.353049:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879479.353053:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879480.057018:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879480.057026:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879480.057034:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879480.057037:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879480.057086:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879480.057088:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879480.057091:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879480.377025:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879480.377041:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879480.377043:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879480.377049:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879480.377052:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879480.377057:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879480.377059:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879480.377062:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879480.377082:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879481.081079:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879481.081088:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879481.081095:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879481.081097:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879481.081135:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879481.081154:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879481.081159:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879481.400976:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879481.400983:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879481.400985:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879481.400990:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879481.400994:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879481.400998:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879481.401001:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879481.401004:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879481.401006:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879482.104983:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879482.104990:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879482.104995:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879482.104997:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879482.105016:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879482.105018:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879482.105021:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879482.361031:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879482.361045:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879482.361049:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879482.361054:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879482.361060:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879482.361071:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10421 00000100:00080000:0.0:1788879482.361097:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879482.361102:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879482.361106:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879482.361116:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879482.361129:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879482.361133:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879482.361136:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879482.361139:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879482.361141:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879482.361147:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879482.361150:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879482.361155:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879482.361159:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879482.361163:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361169:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879482.361173:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879482.361184:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879482.361186:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361190:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879482.361193:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879482.361199:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361215:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879482.361229:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290167680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879482.361245:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879482.361251:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361278:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879482.361283:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879482.361287:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879482.361289:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879482.361292:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879482.361296:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879482.361300:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10421 00000100:00080000:0.0:1788879482.361304:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879482.361307:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879482.361308:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879482.361312:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879482.361313:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879482.361316:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879482.361319:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879482.361321:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879482.361323:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879482.361327:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879482.361330:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879482.361332:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879482.361338:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879482.361341:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361343:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879482.361348:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879482.361351:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361354:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879482.361360:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879482.361362:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361364:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879482.361367:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879482.361370:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361377:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879482.361383:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290167808/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879482.361411:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879482.361414:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361417:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879482.361421:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879482.361427:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879482.361431:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879482.361434:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879482.361439:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879482.361444:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10421 00000100:00080000:0.0:1788879482.361450:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879482.361452:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879482.361454:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879482.361458:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879482.361461:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879482.361465:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879482.361469:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879482.361472:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879482.361474:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879482.361479:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879482.361483:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.361487:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788879482.361488:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879482.361497:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879482.361498:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788879482.361521:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.361523:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879482.361525:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879482.361528:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:0.0:1788879482.361530:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879482.361531:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788879482.361536:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879482.361539:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788879482.361540:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879482.361544:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879482.361546:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879482.361550:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879482.361550:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361557:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879482.361559:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879482.361586:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879482.361587:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290167680:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879482.361595:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.361595:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879482.361602:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:0.0:1788879482.361607:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290167936/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879482.361607:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879482.361630:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879482.361649:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361650:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879482.361655:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361660:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879482.361663:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879482.361666:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879482.361669:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879482.361671:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879482.361674:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879482.361678:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10421 00000100:00080000:0.0:1788879482.361699:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879482.361701:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879482.361702:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879482.361705:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879482.361706:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879482.361708:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879482.361710:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879482.361712:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879482.361713:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879482.361716:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879482.361718:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879482.361719:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879482.361722:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879482.361724:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361725:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879482.361727:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879482.361730:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361731:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879482.361735:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879482.361737:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361739:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879482.361741:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879482.361743:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361748:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879482.361752:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290168064/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879482.361759:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879482.361760:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879482.361762:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.361863:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879482.361866:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:3.0:1788879482.361870:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879482.362003:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.362010:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305094186 00000100:00000001:3.0:1788879482.362023:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879482.362032:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305094186 00000100:00000001:3.0:1788879482.362040:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879482.362046:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879482.362051:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.362056:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879482.362059:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879482.362061:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879482.362064:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879482.362066:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879482.362068:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879482.362073:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290167808:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879482.362078:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879482.362090:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879482.362092:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879482.362093:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879482.362097:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.362105:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879482.362106:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:3.0:1788879482.362111:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879482.362128:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.362132:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305093644 00000100:00000001:3.0:1788879482.362135:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305094186 00000100:00000001:3.0:1788879482.362137:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305093644 00000100:00000001:3.0:1788879482.362140:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305094186 00000100:00000001:3.0:1788879482.362142:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879482.362144:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.362146:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.362149:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879482.362174:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879482.362177:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788879482.362180:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879482.362184:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879482.362185:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879482.362187:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879482.362188:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879482.362192:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879482.362195:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879482.362197:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879482.362201:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290167936:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879482.362206:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879482.362208:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879482.362210:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879482.362212:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879482.362216:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.362222:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879482.362224:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:3.0:1788879482.362228:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879482.362238:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.362242:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305093642 00000100:00000001:3.0:1788879482.362245:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305094186 00000100:00000001:3.0:1788879482.362248:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305093642 00000100:00000001:3.0:1788879482.362251:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305094186 00000100:00000001:3.0:1788879482.362253:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879482.362255:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.362257:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879482.362259:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879482.362260:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879482.362263:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879482.362264:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879482.362267:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879482.362270:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290168064:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879482.362274:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879482.362276:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879482.362278:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879482.362279:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879482.362282:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.362287:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879482.362288:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:3.0:1788879482.362291:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879482.362299:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.362302:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305093642 00000100:00000001:3.0:1788879482.362304:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305094186 00000100:00000001:3.0:1788879482.362307:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305093642 00000100:00000001:3.0:1788879482.362310:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305094186 00000100:00000001:3.0:1788879482.362313:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879482.362315:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.362317:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.362319:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879482.362322:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879482.362324:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788879482.362325:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879482.362327:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879482.362329:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879482.362333:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879482.425000:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879482.425008:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879482.425011:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879482.425017:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879482.425033:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879482.425036:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879482.425039:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879482.425041:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879482.425060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879483.129030:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879483.129038:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879483.129046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879483.129047:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879483.129074:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879483.129078:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879483.129082:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879483.449041:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879483.449050:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879483.449052:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879483.449059:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879483.449064:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879483.449069:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879483.449100:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879483.449103:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879483.449107:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879484.153025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879484.153035:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879484.153043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879484.153049:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879484.153052:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879484.153058:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879484.153063:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879484.153066:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879484.153068:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879484.153071:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879484.153108:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879484.153111:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879484.153114:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879484.472965:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879484.472971:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879484.472973:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879484.472977:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879484.472980:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879484.472984:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879484.472986:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879484.472988:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879484.472990:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879485.176997:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879485.177003:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879485.177007:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879485.177008:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879485.177026:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879485.177029:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879485.177032:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788879485.433136:0:2352:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1788879485.433190:0:2352:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788879485.433206:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879485.433231:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879485.433249:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879485.433252:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879485.433254:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879485.433264:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879485.433269:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879485.433272:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879485.433273:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879485.433276:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879485.433283:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879485.433287:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879485.433292:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879485.433293:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879485.433295:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879485.433296:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879485.433300:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000800:00000100:0.0:1788879485.433304:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788879485.433305:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879485.433324:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879485.433329:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1788879485.433329:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879482/real 1788879485] req@ffff94e508979c00 x1875765290167936/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879485.433333:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879485.433337:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000400:00000001:3.0:1788879485.433346:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879485.433348:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879485.433353:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879485.433356:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879485.433357:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433359:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879485.433361:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879485.433364:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290167936/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879485.433370:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433373:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879485.433375:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433376:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879485.433377:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433379:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879485.433394:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879485.433397:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879485.433401:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879485.433403:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879485.433406:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290167936:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879485.433410:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879485.433414:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879482/real 1788879485] req@ffff94e50897a300 x1875765290168064/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879485.433427:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879485.433428:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879485.433431:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879485.433432:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879485.433434:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433436:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879485.433437:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879485.433441:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290168064/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879485.433448:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433450:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879485.433453:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433455:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879485.433456:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433457:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879485.433473:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879485.433476:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879485.433479:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879485.433482:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879485.433485:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290168064:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879485.433490:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879485.433495:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879482/real 1788879485] req@ffff94e508978e00 x1875765290167680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879485.433504:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879485.433516:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879485.433519:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879485.433521:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879485.433522:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433524:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879485.433526:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879485.433530:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290167680/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879485.433536:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433552:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879485.433555:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433557:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879485.433558:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433560:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879485.433562:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879485.433565:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879485.433569:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879485.433571:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879485.433575:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290167680:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879485.433580:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879485.433595:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879482/real 1788879485] req@ffff94e50897ad80 x1875765290167808/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879485.433604:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879485.433605:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879485.433608:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879485.433610:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879485.433612:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433614:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879485.433616:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879485.433619:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290167808/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879485.433632:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433634:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879485.433636:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433638:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879485.433640:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433641:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879485.433668:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879485.433672:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879485.433675:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879485.433677:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879485.433680:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290167808:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879485.433684:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433687:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879485.433688:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879485.433691:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879485.433697:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879485.433699:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879485.433702:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879485.433706:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879485.433707:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879485.433712:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879485.433713:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433715:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879485.433716:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879485.433717:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879485.433719:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879485.433720:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879485.433721:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879485.433723:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879485.433724:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879485.433727:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879485.433729:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433731:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879485.433732:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879485.433733:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879485.433735:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879485.433736:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879485.433738:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879485.433740:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879485.433741:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879485.433743:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879485.433744:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433746:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879485.433747:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879485.433748:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879485.433750:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879485.433751:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879485.433753:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879485.433755:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879485.433756:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879485.433758:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879485.433760:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433762:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879485.433770:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879485.433772:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879485.433774:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879485.433776:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879485.496996:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879485.497003:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879485.497005:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879485.497010:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879485.497014:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879485.497018:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879485.497021:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879485.497023:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879485.497026:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879486.201065:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879486.201073:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879486.201078:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879486.201079:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879486.201128:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879486.201130:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879486.201133:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879486.520995:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879486.521004:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879486.521008:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879486.521015:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879486.521019:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879486.521025:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879486.521028:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879486.521032:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879486.521036:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879487.225077:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879487.225089:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879487.225092:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879487.225096:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879487.225102:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879487.225105:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879487.225108:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879487.225110:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879487.225115:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879487.225118:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879487.225161:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879487.225165:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879487.225168:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879487.481184:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879487.481198:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879487.481201:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879487.481206:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879487.481212:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879487.481222:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10426 00000100:00080000:0.0:1788879487.481247:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879487.481250:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879487.481252:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879487.481263:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879487.481267:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879487.481271:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879487.481274:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879487.481277:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879487.481278:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879487.481284:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879487.481287:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879487.481302:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879487.481307:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879487.481310:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.481313:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879487.481316:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879487.481327:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879487.481329:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.481333:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879487.481336:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879487.481343:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.481359:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879487.481372:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978700 x1875765290168192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879487.481386:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879487.481392:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.481433:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879487.481437:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879487.481441:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879487.481443:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879487.481445:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879487.481448:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879487.481451:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10426 00000100:00080000:0.0:1788879487.481454:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879487.481456:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879487.481458:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879487.481461:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879487.481463:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879487.481466:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879487.481468:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879487.481470:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879487.481472:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879487.481475:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879487.481478:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879487.481480:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879487.481495:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879487.481498:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.481501:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879487.481507:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879487.481514:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:0.0:1788879487.481515:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.481521:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.481542:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788879487.481547:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879487.481550:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1788879487.481551:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.481561:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879487.481564:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788879487.481565:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879487.481570:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.481573:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.481579:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879487.481585:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:0.0:1788879487.481589:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897aa00 x1875765290168320/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879487.481594:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879487.481614:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.481616:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879487.481626:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879487.481636:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290168192:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879487.481638:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879487.481648:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879487.481652:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1788879487.481660:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:3.0:1788879487.481662:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788879487.481665:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:3.0:1788879487.481669:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879487.481672:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1788879487.481678:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879487.481682:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879487.481693:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10426 02000000:00000001:3.0:1788879487.481694:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879487.481704:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879487.481711:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879487.481714:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879487.481723:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1788879487.481724:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879487.481730:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879487.481732:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000020:00000001:0.0:1788879487.481735:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879487.481743:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1788879487.481745:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879487.481752:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879487.481755:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879487.481763:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879487.481770:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879487.481774:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879487.481779:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879487.481784:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.481788:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879487.481792:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879487.481798:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.481806:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788879487.481808:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:0.0:1788879487.481818:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879487.481825:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.481829:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879487.481830:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:0.0:1788879487.481835:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879487.481842:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1788879487.481843:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.481850:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.481858:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.481858:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879487.481869:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879487.481876:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290168448/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879487.481881:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305099305 00000100:00000001:3.0:1788879487.481958:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879487.481970:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305099306 00000100:00000001:0.0:1788879487.481972:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879487.481983:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.482008:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879487.482012:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879487.482017:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879487.482020:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879487.482023:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879487.482027:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879487.482032:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10426 00000100:00080000:0.0:1788879487.482039:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879487.482043:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879487.482045:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879487.482051:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879487.482054:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879487.482056:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879487.482060:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879487.482064:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879487.482066:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879487.482072:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879487.482078:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.482078:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1788879487.482090:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879487.482094:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879487.482097:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879487.482104:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879487.482106:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.482118:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879487.482118:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000008:00100000:0.0:1788879487.482124:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879487.482134:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879487.482137:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879487/real 1788879487] req@ffff94e508978700 x1875765290168192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879487.482140:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879487.482154:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879487.482158:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.482167:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879487.482170:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:3.0:1788879487.482172:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1788879487.482175:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879487.482178:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879487.482188:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879487.482189:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.482194:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879487.482199:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879487.482207:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290168576/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879487.482208:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879487.482213:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879487.482238:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:3.0:1788879487.482238:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290168192/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879487.482249:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879487.482261:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.482282:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482289:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879487.482294:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482298:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879487.482300:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482305:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879487.482311:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879487.482315:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879487.482321:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879487.482324:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879487.482328:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290168192:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879487.482334:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482337:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879487.482339:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879487.482342:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879487.482348:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879487.482350:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879487.482353:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879487.482357:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879487.482359:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879487.482362:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879487.482364:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482366:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482375:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879487.482379:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.482382:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879487.482385:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879487.482387:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879487.482390:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879487.482392:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879487.482395:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879487.482398:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879487.482401:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879487.482405:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290168320:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879487.482409:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879487.482411:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879487.482413:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879487.482415:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879487.482419:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.482427:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879487.482429:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:3.0:1788879487.482435:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879487.482456:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879487.482462:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879487.482466:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879487.482467:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.482470:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879487.482474:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.482494:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305098764 00000100:00000001:3.0:1788879487.482497:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305099305 00000100:00000001:3.0:1788879487.482499:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305098764 00000100:00000001:3.0:1788879487.482502:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305099306 00000100:00000001:3.0:1788879487.482504:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879487.482506:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.482508:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879487.482513:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879487/real 1788879487] req@ffff94e50897aa00 x1875765290168320/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879487.482522:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879487.482523:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879487.482543:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879487.482546:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879487.482547:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482550:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879487.482551:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879487.482556:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290168320/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879487.482564:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482566:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879487.482568:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482570:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879487.482572:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482573:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879487.482593:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879487.482596:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879487.482601:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879487.482604:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879487.482607:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290168320:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879487.482612:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879487.482613:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879487.482615:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879487.482618:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879487.482620:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879487.482622:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879487.482627:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290168448:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879487.482631:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879487.482633:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879487.482634:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879487.482636:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879487.482640:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.482647:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879487.482648:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:3.0:1788879487.482652:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879487.482665:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879487.482669:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879487.482671:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879487.482673:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.482674:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879487.482677:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.482681:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305098762 00000100:00000001:3.0:1788879487.482684:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305099305 00000100:00000001:3.0:1788879487.482695:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305098762 00000100:00000001:3.0:1788879487.482698:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305099306 00000100:00000001:3.0:1788879487.482700:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879487.482702:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.482705:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879487.482709:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879487/real 1788879487] req@ffff94e508978a80 x1875765290168448/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879487.482718:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879487.482719:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879487.482722:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879487.482724:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879487.482725:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482727:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879487.482729:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879487.482732:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290168448/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879487.482738:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482740:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879487.482748:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482761:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879487.482762:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482763:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879487.482780:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879487.482783:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879487.482787:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879487.482789:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879487.482793:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290168448:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879487.482797:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879487.482799:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879487.482801:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879487.482803:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879487.482805:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879487.482807:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879487.482810:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290168576:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879487.482817:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879487.482819:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879487.482821:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879487.482823:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879487.482826:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.482832:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879487.482834:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:3.0:1788879487.482837:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879487.482850:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879487.482855:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879487.482858:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879487.482860:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.482861:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879487.482864:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.482868:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305098762 00000100:00000001:3.0:1788879487.482871:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305099305 00000100:00000001:3.0:1788879487.482873:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305098762 00000100:00000001:3.0:1788879487.482876:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305099306 00000100:00000001:3.0:1788879487.482878:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879487.482880:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.482882:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879487.482887:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879487/real 1788879487] req@ffff94e50897ad80 x1875765290168576/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879487.482897:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879487.482898:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879487.482931:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879487.482933:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879487.482935:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482937:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879487.482939:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879487.482944:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290168576/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879487.482959:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482961:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879487.482964:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482966:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879487.482967:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.482969:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879487.482983:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879487.482986:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879487.482990:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879487.482992:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879487.482996:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290168576:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879487.483000:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.483002:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879487.483004:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879487.483005:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879487.483008:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879487.483010:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879487.483011:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879487.483014:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879487.483015:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879487.483018:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879487.483019:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.483021:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879487.483022:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879487.483023:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879487.483025:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879487.483026:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879487.483027:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879487.483029:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879487.483030:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879487.483032:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879487.483033:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.483034:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879487.483035:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879487.483037:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879487.483039:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879487.483040:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879487.483041:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879487.483043:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879487.483044:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879487.483047:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879487.483049:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.483051:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879487.483055:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879487.483057:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879487.483059:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879487.483061:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879487.545080:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879487.545091:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879487.545094:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879487.545101:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879487.545106:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879487.545111:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879487.545115:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879487.545119:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879487.545123:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879488.249030:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879488.249038:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879488.249046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879488.249048:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879488.249075:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879488.249078:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879488.249082:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879488.569010:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879488.569020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879488.569022:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879488.569028:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879488.569044:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879488.569047:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879488.569050:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879488.569052:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879488.569055:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879489.272999:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879489.273006:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879489.273013:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879489.273015:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879489.273040:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879489.273043:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879489.273046:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879489.592961:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879489.592967:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879489.592969:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879489.592974:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879489.592977:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879489.592980:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879489.592983:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879489.592986:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879489.592988:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879490.297047:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879490.297057:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879490.297064:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879490.297065:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879490.297144:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879490.297147:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879490.297151:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879490.617039:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879490.617050:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879490.617055:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879490.617065:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879490.617071:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879490.617075:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879490.617079:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879490.617083:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879490.617086:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879491.321047:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879491.321054:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879491.321061:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879491.321063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879491.321093:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879491.321098:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879491.321104:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879491.641015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879491.641025:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879491.641029:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879491.641038:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879491.641045:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879491.641052:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879491.641057:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879491.641063:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879491.641069:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879492.345001:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879492.345009:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879492.345015:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879492.345016:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879492.345041:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879492.345043:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879492.345046:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879492.601010:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879492.601019:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879492.601022:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879492.601025:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879492.601029:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879492.601034:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10431 00000100:00080000:2.0:1788879492.601043:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879492.601045:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879492.601047:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879492.601054:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879492.601056:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879492.601058:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879492.601060:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879492.601062:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879492.601063:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879492.601066:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879492.601068:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879492.601070:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879492.601074:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879492.601076:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879492.601078:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879492.601080:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879492.601089:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879492.601091:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879492.601093:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879492.601096:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879492.601100:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879492.601110:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879492.601118:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290168704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879492.601127:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879492.601131:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879492.601143:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879492.601145:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879492.601148:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879492.601149:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879492.601151:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879492.601153:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879492.601154:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10431 00000100:00080000:2.0:1788879492.601156:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879492.601158:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879492.601158:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879492.601161:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879492.601162:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879492.601163:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879492.601165:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879492.601166:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879492.601167:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879492.601169:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879492.601171:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879492.601172:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879492.601176:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879492.601178:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879492.601179:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879492.601183:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879492.601190:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879492.601192:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879492.601195:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879492.601197:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601198:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879492.601199:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879492.601203:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879492.601204:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601205:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879492.601209:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601210:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879492.601214:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1788879492.601215:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290168832/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879492.601217:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879492.601223:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879492.601227:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879492.601228:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879492.601232:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879492.601232:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879492.601240:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879492.601242:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290168704:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1788879492.601244:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879492.601250:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1788879492.601251:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879492.601254:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879492.601255:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1788879492.601256:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788879492.601258:0:80852:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:3.0:1788879492.601260:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879492.601263:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10431 02000000:00000001:3.0:1788879492.601264:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1788879492.601268:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879492.601270:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788879492.601271:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879492.601272:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879492.601278:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879492.601279:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879492.601280:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879492.601282:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879492.601284:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879492.601285:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879492.601287:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879492.601289:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879492.601290:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879492.601291:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788879492.601292:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879492.601295:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 02000000:00000001:2.0:1788879492.601296:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879492.601301:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879492.601302:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:2.0:1788879492.601304:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879492.601307:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879492.601308:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879492.601312:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879492.601313:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879492.601314:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879492.601315:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879492.601316:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879492.601320:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879492.601323:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290168960/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879492.601328:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879492.601330:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879492.601331:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879492.601333:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879492.601335:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879492.601337:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879492.601339:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1788879492.601340:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879492.601344:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879492.601349:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10431 00000100:00080000:2.0:1788879492.601354:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879492.601354:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879492.601359:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879492.601360:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879492.601361:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879492.601364:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601367:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:2.0:1788879492.601368:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879492.601372:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879492.601372:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879492.601375:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879492.601378:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305104425 00000020:00000001:2.0:1788879492.601381:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879492.601385:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879492.601387:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879492.601388:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1788879492.601394:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879492.601398:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879492.601400:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879492.601403:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879492.601405:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601405:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305104425 00000100:00000001:2.0:1788879492.601411:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879492.601414:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879492.601414:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000008:00000001:2.0:1788879492.601417:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879492.601419:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879492.601423:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879492.601426:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601427:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879492.601428:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879492.601432:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879492.601433:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879492.601435:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601436:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879492.601441:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879492.601444:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879492/real 1788879492] req@ffff94e5045e5180 x1875765290168704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879492.601449:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290169088/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788879492.601458:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879492.601460:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879492.601463:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879492.601467:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879492.601468:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879492.601470:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879492.601472:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601474:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879492.601477:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601477:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879492.601485:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290168704/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879492.601522:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.601526:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879492.601529:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.601531:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879492.601542:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.601544:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879492.601546:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879492.601549:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879492.601553:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879492.601555:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879492.601559:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290168704:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879492.601563:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.601565:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879492.601567:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879492.601594:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879492.601599:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879492.601601:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879492.601604:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879492.601610:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879492.601613:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879492.601616:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879492.601618:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.601621:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.601630:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879492.601633:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601635:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879492.601653:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879492.601655:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879492.601656:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879492.601657:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879492.601659:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879492.601660:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879492.601662:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879492.601664:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290168832:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879492.601667:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879492.601668:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879492.601669:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879492.601671:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879492.601673:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601678:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879492.601679:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:3.0:1788879492.601682:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879492.601692:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879492.601695:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879492.601696:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879492.601698:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601699:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879492.601701:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601704:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305103883 00000100:00000001:3.0:1788879492.601706:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305104425 00000100:00000001:3.0:1788879492.601708:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305103883 00000100:00000001:3.0:1788879492.601710:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305104425 00000100:00000001:3.0:1788879492.601712:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879492.601713:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601714:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879492.601717:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879492/real 1788879492] req@ffff94e5045e5880 x1875765290168832/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879492.601723:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879492.601724:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879492.601725:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879492.601726:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879492.601727:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.601728:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879492.601729:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879492.601731:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290168832/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879492.601736:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.601737:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879492.601739:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.601740:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879492.601741:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.601742:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879492.601774:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879492.601777:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879492.601781:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879492.601784:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879492.601787:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290168832:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879492.601792:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879492.601794:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879492.601796:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879492.601799:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879492.601801:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879492.601804:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879492.601808:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290168960:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879492.601812:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879492.601830:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879492.601843:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879492.601845:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879492.601847:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601851:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879492.601852:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:3.0:1788879492.601856:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879492.601865:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879492.601868:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879492.601869:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879492.601871:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601872:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879492.601874:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601877:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305103881 00000100:00000001:3.0:1788879492.601879:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305104425 00000100:00000001:3.0:1788879492.601881:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305103881 00000100:00000001:3.0:1788879492.601883:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305104425 00000100:00000001:3.0:1788879492.601885:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879492.601887:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.601888:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879492.601891:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879492/real 1788879492] req@ffff94e5045e6a00 x1875765290168960/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879492.601897:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879492.601898:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879492.601899:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879492.601901:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879492.601902:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.601903:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879492.601904:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879492.601907:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290168960/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879492.601923:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.601925:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879492.601927:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.601928:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879492.601930:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.601931:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879492.601960:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879492.601995:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879492.601998:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879492.601999:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879492.602001:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290168960:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879492.602004:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879492.602005:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879492.602006:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879492.602007:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879492.602009:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879492.602010:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879492.602012:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290169088:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879492.602014:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879492.602015:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879492.602017:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879492.602018:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879492.602020:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.602024:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879492.602025:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:3.0:1788879492.602027:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879492.602035:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879492.602038:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879492.602040:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879492.602041:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.602074:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879492.602076:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.602079:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305103882 00000100:00000001:3.0:1788879492.602080:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305104425 00000100:00000001:3.0:1788879492.602082:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305103882 00000100:00000001:3.0:1788879492.602083:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305104425 00000100:00000001:3.0:1788879492.602085:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879492.602086:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.602087:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879492.602090:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879492/real 1788879492] req@ffff94e5045e6680 x1875765290169088/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879492.602094:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879492.602095:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879492.602096:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879492.602097:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879492.602098:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.602099:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879492.602100:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879492.602103:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290169088/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879492.602107:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.602108:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879492.602110:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.602111:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879492.602111:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.602113:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879492.602126:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879492.602127:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879492.602129:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879492.602130:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879492.602132:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290169088:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879492.602135:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.602136:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879492.602137:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879492.602138:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879492.602140:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879492.602141:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879492.602141:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879492.602143:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879492.602145:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879492.602146:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879492.602148:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.602149:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879492.602151:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879492.602152:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879492.602154:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879492.602155:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879492.602156:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879492.602158:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879492.602159:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879492.602160:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879492.602161:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.602163:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879492.602175:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879492.602176:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879492.602195:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879492.602197:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879492.602199:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879492.602202:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879492.602204:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879492.602208:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879492.602210:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.602214:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879492.602220:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879492.602224:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879492.602228:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879492.602231:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879492.665001:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879492.665007:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879492.665009:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879492.665013:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879492.665015:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879492.665019:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879492.665021:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879492.665023:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879492.665025:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879493.369066:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879493.369074:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879493.369081:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879493.369082:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879493.369111:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879493.369114:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879493.369118:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879493.689003:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879493.689011:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879493.689014:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879493.689020:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879493.689025:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879493.689029:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879493.689033:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879493.689035:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879493.689039:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879494.392983:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879494.392987:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879494.392992:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879494.392995:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879494.392997:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879494.393000:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879494.393002:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879494.393004:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879494.393006:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879494.393007:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879494.393023:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879494.393025:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879494.393027:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879494.713007:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879494.713015:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879494.713019:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879494.713025:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879494.713030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879494.713035:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879494.713039:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879494.713044:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879494.713049:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879495.417027:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879495.417035:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879495.417041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879495.417043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879495.417092:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879495.417096:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879495.417100:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879495.737027:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879495.737037:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879495.737040:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879495.737046:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879495.737051:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879495.737057:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879495.737061:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879495.737064:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879495.737068:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879496.441020:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879496.441029:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879496.441036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879496.441038:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879496.441065:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879496.441069:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879496.441073:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879496.761009:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879496.761017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879496.761021:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879496.761028:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879496.761044:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879496.761049:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879496.761066:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879496.761069:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879496.761073:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879497.465000:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879497.465010:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879497.465012:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879497.465016:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879497.465020:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879497.465023:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879497.465024:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879497.465026:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879497.465029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879497.465030:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879497.465053:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879497.465055:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879497.465058:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879497.721048:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879497.721064:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879497.721069:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879497.721074:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879497.721080:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879497.721096:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10436 00000100:00080000:3.0:1788879497.721109:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879497.721112:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879497.721113:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879497.721122:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879497.721125:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879497.721128:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879497.721131:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879497.721133:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879497.721134:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879497.721138:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879497.721140:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879497.721143:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879497.721146:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879497.721148:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721150:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879497.721153:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879497.721167:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879497.721168:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721172:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879497.721174:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879497.721179:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721192:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879497.721202:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290169216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879497.721213:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879497.721217:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721237:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879497.721240:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879497.721245:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879497.721247:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879497.721248:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879497.721251:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879497.721253:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10436 00000100:00080000:3.0:1788879497.721256:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879497.721258:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879497.721260:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879497.721262:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879497.721264:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879497.721266:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879497.721268:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879497.721270:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879497.721271:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879497.721274:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879497.721276:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879497.721277:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879497.721283:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879497.721285:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721287:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879497.721288:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879497.721297:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00100000:3.0:1788879497.721298:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879497.721303:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00000001:3.0:1788879497.721303:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721307:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.721310:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788879497.721312:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879497.721315:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721318:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879497.721319:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879497.721320:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879497.721324:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721335:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879497.721336:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879497.721345:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788879497.721345:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290169344/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788879497.721358:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721376:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879497.721378:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879497.721380:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290169216:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879497.721381:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879497.721385:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879497.721395:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879497.721399:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879497.721404:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879497.721408:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879497.721408:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879497.721414:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10436 00000100:00080000:3.0:1788879497.721418:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1788879497.721419:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788879497.721420:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879497.721425:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879497.721428:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879497.721429:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879497.721432:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879497.721435:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879497.721440:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 02000000:00000001:2.0:1788879497.721441:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879497.721444:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879497.721447:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879497.721449:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:2.0:1788879497.721465:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721466:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879497.721475:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879497.721478:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879497.721480:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721482:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879497.721485:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879497.721487:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721489:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.721491:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879497.721507:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 02000000:00000001:3.0:1788879497.721509:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879497.721512:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.721513:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879497.721515:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879497.721520:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879497.721522:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721528:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879497.721533:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290169472/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879497.721540:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879497.721543:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721545:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879497.721547:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879497.721550:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879497.721552:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879497.721554:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879497.721556:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879497.721559:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10436 00000100:00080000:3.0:1788879497.721563:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879497.721565:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879497.721566:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879497.721571:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879497.721572:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879497.721574:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879497.721581:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000800:00000100:2.0:1788879497.721582:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879497.721586:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879497.721588:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879497.721594:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879497.721597:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879497.721599:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879497.721601:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879497.721603:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788879497.721604:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721608:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879497.721611:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879497.721615:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.721616:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879497.721617:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.721623:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879497.721623:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879497.721627:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721630:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879497.721631:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879497.721632:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879497.721636:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721641:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.721643:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879497.721647:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290169600/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879497.721653:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305109545 00000100:00000001:3.0:1788879497.721655:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879497.721661:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879497.721664:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.721671:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788879497.721690:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305109545 00000100:00000001:2.0:1788879497.721697:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879497.721704:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879497.721719:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.721722:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879497.721732:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879497/real 1788879497] req@ffff94e50449c700 x1875765290169216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879497.721746:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879497.721749:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879497.721755:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879497.721760:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879497.721763:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.721766:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879497.721769:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879497.721775:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290169216/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879497.721784:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.721787:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879497.721790:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.721792:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879497.721793:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.721796:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879497.721798:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879497.721802:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879497.721808:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879497.721811:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879497.721816:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290169216:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879497.721821:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.721824:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879497.721825:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879497.721828:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879497.721832:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879497.721835:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879497.721837:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879497.721842:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879497.721844:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879497.721847:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879497.721849:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.721852:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.721860:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879497.721862:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.721865:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879497.721867:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879497.721868:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879497.721870:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879497.721872:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879497.721875:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879497.721877:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879497.721880:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879497.721884:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290169344:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879497.721888:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879497.721891:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879497.721892:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879497.721894:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879497.721897:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.721928:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879497.721930:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:2.0:1788879497.721935:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879497.721952:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879497.721957:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879497.721960:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879497.721962:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.721964:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879497.721967:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.721972:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305109004 00000100:00000001:2.0:1788879497.721975:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305109545 00000100:00000001:2.0:1788879497.721977:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305109004 00000100:00000001:2.0:1788879497.721991:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305109545 00000100:00000001:2.0:1788879497.721994:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879497.721996:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.721998:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879497.722003:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879497/real 1788879497] req@ffff94e50449f480 x1875765290169344/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879497.722013:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879497.722015:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879497.722018:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879497.722019:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879497.722021:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722023:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879497.722024:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879497.722029:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290169344/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879497.722036:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722039:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879497.722042:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722043:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879497.722052:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722054:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879497.722073:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879497.722077:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879497.722081:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879497.722083:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879497.722087:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290169344:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879497.722091:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879497.722093:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879497.722095:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879497.722098:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879497.722099:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879497.722102:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879497.722105:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290169472:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879497.722108:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879497.722111:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879497.722113:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879497.722115:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879497.722118:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.722124:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879497.722125:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:2.0:1788879497.722129:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879497.722141:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879497.722146:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879497.722148:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879497.722150:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.722152:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879497.722155:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.722158:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305109002 00000100:00000001:2.0:1788879497.722161:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305109545 00000100:00000001:2.0:1788879497.722164:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305109002 00000100:00000001:2.0:1788879497.722166:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305109545 00000100:00000001:2.0:1788879497.722168:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879497.722171:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.722173:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879497.722177:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879497/real 1788879497] req@ffff94e50449f800 x1875765290169472/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879497.722193:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879497.722195:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879497.722197:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879497.722199:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879497.722201:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722203:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879497.722204:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879497.722208:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290169472/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879497.722215:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722218:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879497.722220:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722222:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879497.722224:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722226:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879497.722242:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879497.722245:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879497.722248:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879497.722251:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879497.722255:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290169472:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879497.722259:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879497.722261:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879497.722262:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879497.722265:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879497.722267:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879497.722269:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879497.722272:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290169600:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879497.722276:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879497.722278:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879497.722280:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879497.722282:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879497.722285:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.722290:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879497.722292:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:2.0:1788879497.722295:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879497.722308:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879497.722312:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879497.722315:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879497.722316:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.722318:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879497.722320:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.722324:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305109002 00000100:00000001:2.0:1788879497.722327:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305109545 00000100:00000001:2.0:1788879497.722339:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305109002 00000100:00000001:2.0:1788879497.722342:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305109545 00000100:00000001:2.0:1788879497.722344:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879497.722346:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.722348:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879497.722352:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879497/real 1788879497] req@ffff94e50449d880 x1875765290169600/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879497.722360:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879497.722361:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879497.722364:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879497.722365:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879497.722367:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722369:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879497.722370:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879497.722374:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290169600/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879497.722381:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722383:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879497.722386:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722387:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879497.722389:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722390:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879497.722406:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879497.722409:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879497.722413:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879497.722415:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879497.722418:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290169600:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879497.722422:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722424:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879497.722425:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879497.722427:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879497.722429:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879497.722431:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879497.722432:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879497.722435:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879497.722436:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879497.722438:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879497.722439:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722441:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879497.722442:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879497.722443:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879497.722445:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879497.722446:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879497.722447:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879497.722449:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879497.722450:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879497.722468:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879497.722470:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722472:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879497.722473:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879497.722474:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879497.722476:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879497.722478:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879497.722479:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879497.722481:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879497.722482:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879497.722484:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879497.722485:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722487:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879497.722490:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879497.722492:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879497.722494:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879497.722496:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879497.785014:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879497.785018:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879497.785035:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879497.785040:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879497.785044:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879497.785048:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879497.785050:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879497.785053:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879497.785055:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879498.489021:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879498.489028:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879498.489034:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879498.489035:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879498.489059:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879498.489062:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879498.489065:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879498.808980:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879498.808986:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879498.808989:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879498.808993:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879498.808996:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879498.809000:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879498.809002:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879498.809004:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879498.809006:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879499.513030:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879499.513036:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879499.513040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879499.513041:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879499.513061:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879499.513063:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879499.513066:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879499.832975:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879499.832982:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879499.832985:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879499.832989:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879499.832992:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879499.832996:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879499.832998:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879499.833000:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879499.833002:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879500.537965:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879500.537972:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879500.537976:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879500.537977:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879500.538000:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879500.538003:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879500.538005:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879500.856954:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879500.856961:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879500.856963:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879500.856967:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879500.856970:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879500.856974:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879500.856976:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879500.856978:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879500.856981:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879501.561004:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879501.561029:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879501.561040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879501.561043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879501.561078:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879501.561084:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879501.561089:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879501.880966:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879501.880972:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879501.880974:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879501.880979:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879501.880982:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879501.880985:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879501.880988:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879501.880990:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879501.880992:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879502.585985:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879502.585993:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879502.586000:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879502.586001:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879502.586026:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879502.586029:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879502.586033:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879502.841008:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879502.841019:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879502.841022:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879502.841027:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879502.841033:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879502.841042:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10441 00000100:00080000:3.0:1788879502.841055:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879502.841059:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879502.841062:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879502.841072:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879502.841075:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879502.841080:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879502.841083:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879502.841086:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879502.841088:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879502.841094:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879502.841097:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879502.841101:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879502.841106:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879502.841110:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879502.841114:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879502.841117:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879502.841129:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879502.841131:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879502.841135:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879502.841139:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879502.841145:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879502.841160:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879502.841171:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290169728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879502.841185:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879502.841191:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879502.841212:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879502.841217:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879502.841221:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879502.841223:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879502.841225:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879502.841229:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879502.841232:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10441 00000100:00080000:3.0:1788879502.841236:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879502.841238:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879502.841240:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879502.841244:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879502.841245:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879502.841248:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879502.841250:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879502.841253:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879502.841254:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879502.841258:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879502.841261:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879502.841264:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879502.841272:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879502.841275:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879502.841277:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879502.841284:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879502.841288:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879502.841291:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879502.841296:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879502.841297:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879502.841300:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879502.841302:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879502.841304:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788879502.841305:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.841318:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879502.841319:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.841324:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879502.841328:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879502.841331:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879502.841333:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290169856/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788879502.841348:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879502.841355:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879502.841358:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788879502.841361:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879502.841367:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879502.841378:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290169728:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879502.841380:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879502.841391:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879502.841396:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788879502.841405:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879502.841409:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879502.841411:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879502.841415:0:178195:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:2.0:1788879502.841430:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879502.841435:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879502.841439:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788879502.841446:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10441 02000000:00000001:2.0:1788879502.841452:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879502.841453:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879502.841459:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879502.841462:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879502.841466:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879502.841468:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879502.841472:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879502.841475:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879502.841478:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879502.841480:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879502.841486:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788879502.841486:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879502.841495:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:3.0:1788879502.841496:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879502.841508:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879502.841511:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788879502.841516:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879502.841524:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879502.841528:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879502.841533:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879502.841539:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879502.841541:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879502.841551:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879502.841554:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879502.841559:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879502.841563:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000800:00000100:2.0:1788879502.841569:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879502.841571:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.841590:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879502.841590:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.841602:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879502.841606:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.841609:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:3.0:1788879502.841611:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290169984/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879502.841616:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.841630:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305114665 00000100:00000001:3.0:1788879502.841639:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879502.841643:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879502.841647:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879502.841651:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879502.841652:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788879502.841664:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879502.841667:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879502.841670:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879502.841674:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879502.841678:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10441 00000100:00000001:2.0:1788879502.841684:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305114665 00000100:00080000:3.0:1788879502.841685:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879502.841694:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879502.841697:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879502.841699:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:3.0:1788879502.841716:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879502.841718:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788879502.841724:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879502.841727:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879502.841731:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879502.841733:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1788879502.841744:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879502.841748:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1788879502.841757:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879502/real 1788879502] req@ffff94e50449fb80 x1875765290169728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879502.841757:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879502.841777:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879502.841786:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879502.841788:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879502.841791:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788879502.841791:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879502.841799:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879502.841800:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879502.841807:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:3.0:1788879502.841808:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.841814:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879502.841815:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879502.841824:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879502.841828:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.841831:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879502.841832:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.841837:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879502.841842:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:2.0:1788879502.841849:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290169728/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879502.841850:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879502.841860:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879502.841868:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879502.841876:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.841878:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.841889:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879502.841891:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.841894:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.841898:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879502.841941:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.841945:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00100000:3.0:1788879502.841948:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290170112/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879502.841949:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879502.841965:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879502.841987:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879502.841988:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879502.841999:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.842000:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788879502.842013:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879502.842015:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290169728:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879502.842029:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842032:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879502.842034:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879502.842037:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879502.842041:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879502.842044:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879502.842046:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879502.842051:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879502.842054:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879502.842058:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879502.842060:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842062:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842071:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879502.842073:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.842075:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879502.842077:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879502.842079:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879502.842080:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879502.842082:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879502.842083:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879502.842085:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879502.842087:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879502.842089:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290169856:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879502.842092:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879502.842094:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879502.842095:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879502.842096:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879502.842099:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.842104:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879502.842106:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:2.0:1788879502.842110:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879502.842122:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879502.842125:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879502.842127:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879502.842128:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.842129:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879502.842131:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.842134:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305114124 00000100:00000001:2.0:1788879502.842137:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305114665 00000100:00000001:2.0:1788879502.842138:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305114124 00000100:00000001:2.0:1788879502.842140:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305114665 00000100:00000001:2.0:1788879502.842141:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879502.842143:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.842144:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879502.842148:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879502/real 1788879502] req@ffff94e50449c380 x1875765290169856/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879502.842154:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879502.842155:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879502.842157:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879502.842158:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879502.842159:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842161:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879502.842162:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879502.842164:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290169856/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879502.842168:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842170:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879502.842171:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842172:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879502.842173:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842174:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879502.842187:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879502.842189:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879502.842191:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879502.842193:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879502.842195:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290169856:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879502.842197:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879502.842198:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879502.842199:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879502.842201:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879502.842202:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879502.842203:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879502.842206:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290169984:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879502.842211:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879502.842212:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879502.842213:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879502.842214:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879502.842217:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.842221:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879502.842222:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:2.0:1788879502.842224:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879502.842230:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879502.842233:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879502.842234:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879502.842235:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.842237:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879502.842238:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.842241:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305114122 00000100:00000001:2.0:1788879502.842243:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305114665 00000100:00000001:2.0:1788879502.842245:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305114122 00000100:00000001:2.0:1788879502.842246:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305114665 00000100:00000001:2.0:1788879502.842248:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879502.842249:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.842250:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879502.842252:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879502/real 1788879502] req@ffff94e50449d180 x1875765290169984/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879502.842257:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879502.842258:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879502.842259:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879502.842260:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879502.842261:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842262:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879502.842262:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879502.842264:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290169984/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879502.842274:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842276:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879502.842277:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842278:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879502.842279:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842280:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879502.842290:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879502.842291:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879502.842293:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879502.842294:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879502.842296:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290169984:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879502.842299:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879502.842300:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879502.842301:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879502.842303:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879502.842304:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879502.842306:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879502.842307:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290170112:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879502.842309:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879502.842310:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879502.842311:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879502.842312:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879502.842315:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.842318:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879502.842319:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:2.0:1788879502.842321:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879502.842326:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879502.842328:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879502.842329:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879502.842330:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.842332:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879502.842334:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.842336:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305114122 00000100:00000001:2.0:1788879502.842338:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305114665 00000100:00000001:2.0:1788879502.842340:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305114122 00000100:00000001:2.0:1788879502.842341:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305114665 00000100:00000001:2.0:1788879502.842343:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879502.842344:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.842345:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879502.842347:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879502/real 1788879502] req@ffff94e50449ce00 x1875765290170112/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879502.842352:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879502.842352:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879502.842353:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879502.842354:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879502.842355:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842356:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879502.842357:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879502.842359:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290170112/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879502.842363:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842365:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879502.842367:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842368:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879502.842369:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842370:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879502.842380:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879502.842381:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879502.842383:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879502.842384:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879502.842387:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290170112:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879502.842390:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842391:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879502.842392:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879502.842393:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879502.842395:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879502.842395:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879502.842396:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879502.842398:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879502.842398:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879502.842400:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879502.842401:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842402:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879502.842403:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879502.842404:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879502.842405:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879502.842406:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879502.842407:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879502.842408:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879502.842409:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879502.842410:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879502.842411:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842412:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879502.842412:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879502.842413:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879502.842414:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879502.842414:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879502.842415:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879502.842433:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879502.842434:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879502.842436:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879502.842437:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842439:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879502.842442:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879502.842444:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879502.842445:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879502.842447:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879502.904983:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879502.904992:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879502.904996:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879502.905004:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879502.905010:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879502.905015:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879502.905020:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879502.905025:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879502.905030:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879503.609032:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879503.609040:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879503.609047:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879503.609049:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879503.609083:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879503.609086:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879503.609090:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879503.929004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879503.929012:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879503.929015:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879503.929022:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879503.929026:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879503.929031:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879503.929035:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879503.929039:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879503.929042:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879504.633099:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879504.633108:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879504.633115:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879504.633119:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879504.633141:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879504.633146:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879504.633150:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879504.633153:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879504.633157:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879504.633161:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879504.633237:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879504.633240:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879504.633245:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879504.952998:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879504.953006:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879504.953010:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879504.953016:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879504.953021:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879504.953027:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879504.953031:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879504.953034:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879504.953037:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879505.657049:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879505.657057:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879505.657064:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879505.657066:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879505.657093:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879505.657097:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879505.657101:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879505.977019:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879505.977027:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879505.977031:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879505.977037:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879505.977042:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879505.977047:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879505.977050:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879505.977054:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879505.977057:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879506.681045:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879506.681054:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879506.681062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879506.681064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879506.681091:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879506.681094:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879506.681098:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879507.001015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879507.001023:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879507.001026:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879507.001033:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879507.001037:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879507.001043:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879507.001047:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879507.001050:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879507.001053:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879507.705031:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879507.705041:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879507.705045:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879507.705049:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879507.705056:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879507.705059:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879507.705062:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879507.705065:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879507.705070:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879507.705072:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879507.705099:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879507.705102:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879507.705105:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879507.960996:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879507.961008:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879507.961011:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879507.961017:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879507.961023:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879507.961032:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10446 00000100:00080000:3.0:1788879507.961045:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879507.961050:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879507.961053:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879507.961063:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879507.961067:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879507.961072:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879507.961075:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879507.961078:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879507.961080:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879507.961086:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879507.961089:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879507.961093:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879507.961097:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879507.961101:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879507.961105:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879507.961108:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879507.961132:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879507.961135:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879507.961139:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879507.961143:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879507.961149:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879507.961174:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879507.961186:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290170240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879507.961200:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879507.961206:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879507.961229:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879507.961233:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879507.961238:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879507.961240:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879507.961242:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879507.961246:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879507.961249:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10446 00000100:00080000:3.0:1788879507.961254:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879507.961257:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879507.961259:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879507.961264:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879507.961266:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879507.961269:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879507.961273:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879507.961276:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879507.961279:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879507.961283:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879507.961287:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879507.961289:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879507.961304:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788879507.961305:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879507.961314:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.961315:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879507.961322:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879507.961324:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1788879507.961330:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879507.961332:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879507.961338:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788879507.961340:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879507.961347:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879507.961348:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879507.961356:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879507.961357:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879507.961366:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879507.961367:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879507.961377:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879507.961399:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879507.961402:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879507.961409:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290170240:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879507.961413:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.961435:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1788879507.961442:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290170368/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788879507.961445:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879507.961452:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879507.961462:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879507.961469:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788879507.961473:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879507.961474:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879507.961487:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879507.961491:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879507.961496:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879507.961499:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879507.961501:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879507.961503:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879507.961505:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879507.961509:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000100:00080000:3.0:1788879507.961513:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10446 00000100:00000001:2.0:1788879507.961520:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879507.961522:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879507.961528:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879507.961530:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879507.961535:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879507.961537:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879507.961540:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879507.961544:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879507.961547:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879507.961549:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879507.961555:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879507.961559:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879507.961562:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879507.961566:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879507.961569:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879507.961571:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879507.961577:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1788879507.961578:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00000001:3.0:1788879507.961586:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879507.961592:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.961600:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788879507.961601:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879507.961609:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.961612:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879507.961613:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879507.961620:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879507.961622:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879507.961624:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.961628:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879507.961635:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.961638:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.961647:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305119785 00000100:00100000:3.0:1788879507.961649:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290170496/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879507.961663:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879507.961665:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879507.961673:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879507.961678:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879507.961681:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879507.961685:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879507.961686:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879507.961688:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879507.961691:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879507.961694:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305119785 00000100:00080000:3.0:1788879507.961695:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10446 00000100:00000001:2.0:1788879507.961709:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788879507.961710:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879507.961720:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879507.961722:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879507.961723:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879507.961736:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879507.961736:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879507.961741:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879507.961744:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1788879507.961746:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879507.961752:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879507.961758:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1788879507.961760:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879507/real 1788879507] req@ffff94e50449e680 x1875765290170240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879507.961761:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879507.961776:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879507.961786:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879507.961795:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879507.961798:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788879507.961804:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879507.961810:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:3.0:1788879507.961812:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879507.961818:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:3.0:1788879507.961819:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.961826:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879507.961827:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879507.961832:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879507.961836:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:3.0:1788879507.961840:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.961842:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879507.961846:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879507.961856:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290170240/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879507.961857:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879507.961862:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879507.961870:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879507.961875:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879507.961880:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.961887:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879507.961892:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.961895:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879507.961951:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879507.961952:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290170624/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879507.961963:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879507.961981:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879507.961982:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879507.961989:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.961993:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879507.961996:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.962005:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879507.962016:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879507.962030:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879507.962047:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879507.962055:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290170240:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879507.962065:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962070:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879507.962073:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879507.962078:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879507.962084:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879507.962087:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879507.962090:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879507.962095:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879507.962097:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879507.962101:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879507.962103:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962106:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962114:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879507.962117:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.962120:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879507.962122:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879507.962124:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879507.962126:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879507.962128:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879507.962131:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879507.962135:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879507.962138:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879507.962142:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290170368:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879507.962147:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879507.962150:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879507.962152:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879507.962166:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879507.962170:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.962180:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879507.962182:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:2.0:1788879507.962188:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879507.962208:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879507.962214:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879507.962217:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879507.962219:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.962222:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879507.962226:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.962231:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305119244 00000100:00000001:2.0:1788879507.962235:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305119785 00000100:00000001:2.0:1788879507.962238:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305119244 00000100:00000001:2.0:1788879507.962241:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305119785 00000100:00000001:2.0:1788879507.962244:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879507.962246:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.962249:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879507.962254:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879507/real 1788879507] req@ffff94e50449df80 x1875765290170368/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879507.962265:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879507.962267:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879507.962270:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879507.962272:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879507.962274:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962276:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879507.962278:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879507.962283:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290170368/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879507.962292:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962295:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879507.962298:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962300:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879507.962302:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962304:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879507.962335:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879507.962338:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879507.962342:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879507.962345:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879507.962349:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290170368:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879507.962353:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879507.962355:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879507.962356:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879507.962359:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879507.962362:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879507.962364:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879507.962368:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290170496:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879507.962371:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879507.962373:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879507.962375:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879507.962377:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879507.962405:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.962415:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879507.962416:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:2.0:1788879507.962421:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879507.962438:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879507.962444:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879507.962448:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879507.962451:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.962453:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879507.962456:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.962460:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305119242 00000100:00000001:2.0:1788879507.962463:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305119785 00000100:00000001:2.0:1788879507.962466:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305119242 00000100:00000001:2.0:1788879507.962469:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305119785 00000100:00000001:2.0:1788879507.962471:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879507.962474:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.962476:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879507.962481:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879507/real 1788879507] req@ffff94e50449ce00 x1875765290170496/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879507.962490:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879507.962492:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879507.962495:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879507.962497:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879507.962499:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962501:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879507.962503:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879507.962507:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290170496/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879507.962516:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962518:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879507.962521:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962523:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879507.962525:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962527:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879507.962545:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879507.962547:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879507.962551:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879507.962553:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879507.962558:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290170496:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879507.962562:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879507.962564:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879507.962566:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879507.962569:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879507.962571:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879507.962573:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879507.962577:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290170624:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879507.962580:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879507.962583:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879507.962585:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879507.962587:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879507.962590:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.962594:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879507.962596:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:2.0:1788879507.962600:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879507.962611:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879507.962616:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879507.962618:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879507.962620:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.962622:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879507.962625:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.962629:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305119242 00000100:00000001:2.0:1788879507.962632:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305119785 00000100:00000001:2.0:1788879507.962635:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305119242 00000100:00000001:2.0:1788879507.962637:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305119785 00000100:00000001:2.0:1788879507.962640:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879507.962642:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.962644:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879507.962648:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879507/real 1788879507] req@ffff94e50449d180 x1875765290170624/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879507.962666:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879507.962668:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879507.962671:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879507.962673:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879507.962674:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962676:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879507.962678:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879507.962682:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290170624/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879507.962690:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962692:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879507.962695:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962697:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879507.962698:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962700:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879507.962717:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879507.962720:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879507.962724:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879507.962727:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879507.962730:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290170624:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879507.962734:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962737:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879507.962738:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879507.962740:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879507.962742:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879507.962744:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879507.962746:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879507.962748:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879507.962750:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879507.962753:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879507.962754:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962757:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879507.962758:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879507.962760:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879507.962762:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879507.962764:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879507.962765:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879507.962768:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879507.962769:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879507.962771:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879507.962773:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962774:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879507.962776:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879507.962777:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879507.962779:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879507.962780:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879507.962781:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879507.962783:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879507.962784:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879507.962786:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879507.962787:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962789:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879507.962793:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879507.962795:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879507.962797:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879507.962799:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879508.025020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879508.025027:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879508.025030:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879508.025037:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879508.025042:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879508.025046:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879508.025050:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879508.025053:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879508.025056:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879508.729005:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879508.729013:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879508.729020:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879508.729022:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879508.729066:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879508.729070:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879508.729074:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879509.048987:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879509.048997:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879509.049001:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879509.049008:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879509.049013:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879509.049018:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879509.049022:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879509.049026:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879509.049029:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879509.753040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879509.753048:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879509.753055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879509.753057:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879509.753114:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879509.753118:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879509.753122:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879510.073022:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879510.073031:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879510.073034:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879510.073040:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879510.073045:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879510.073051:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879510.073055:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879510.073058:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879510.073062:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879510.777009:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879510.777016:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879510.777023:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879510.777024:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879510.777057:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879510.777061:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879510.777064:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879511.097006:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879511.097013:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879511.097016:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879511.097022:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879511.097027:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879511.097032:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879511.097035:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879511.097038:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879511.097041:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879511.801071:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879511.801079:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879511.801085:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879511.801087:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879511.801195:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879511.801199:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879511.801203:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879512.121015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879512.121022:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879512.121024:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879512.121029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879512.121033:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879512.121038:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879512.121041:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879512.121044:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879512.121046:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879512.825023:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879512.825028:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879512.825033:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879512.825034:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879512.825070:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879512.825073:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879512.825076:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879513.081056:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879513.081093:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879513.081098:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879513.081104:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879513.081126:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879513.081134:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10451 00000100:00080000:2.0:1788879513.081147:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879513.081151:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879513.081154:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879513.081163:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879513.081166:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879513.081171:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879513.081174:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879513.081178:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879513.081179:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879513.081208:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879513.081214:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879513.081220:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879513.081226:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879513.081232:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879513.081237:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879513.081241:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879513.081258:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879513.081261:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879513.081267:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879513.081272:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879513.081279:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879513.081314:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879513.081329:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290170752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879513.081366:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879513.081373:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879513.081398:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879513.081405:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879513.081410:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879513.081414:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879513.081417:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879513.081423:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879513.081428:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10451 00000100:00000001:3.0:1788879513.081433:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788879513.081436:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879513.081439:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879513.081443:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879513.081445:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879513.081450:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879513.081451:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879513.081455:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879513.081461:0:80852:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:3.0:1788879513.081462:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1788879513.081468:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:3.0:1788879513.081469:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879513.081476:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879513.081477:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00100000:3.0:1788879513.081484:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290170752:192.168.203.129@tcp:250:kworker.0 00000020:00000001:2.0:1788879513.081487:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879513.081495:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788879513.081497:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788879513.081501:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788879513.081503:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:3.0:1788879513.081507:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879513.081511:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1788879513.081515:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:3.0:1788879513.081519:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879513.081522:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879513.081528:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879513.081536:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879513.081541:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788879513.081542:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.081547:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:2.0:1788879513.081550:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879513.081553:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:2.0:1788879513.081558:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879513.081565:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879513.081569:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879513.081577:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879513.081579:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879513.081583:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879513.081586:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879513.081590:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879513.081601:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788879513.081606:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:2.0:1788879513.081617:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290170880/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879513.081638:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879513.081649:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879513.081653:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.081654:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879513.081659:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879513.081661:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.081662:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:2.0:1788879513.081669:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879513.081669:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.081677:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305124905 00000100:00080000:2.0:1788879513.081678:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879513.081683:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879513.081686:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879513.081688:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788879513.081691:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879513.081715:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10451 00000100:00080000:2.0:1788879513.081721:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879513.081722:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305124905 00000100:00000001:2.0:1788879513.081728:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879513.081731:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879513.081737:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879513.081739:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879513.081746:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879513.081749:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879513.081751:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:2.0:1788879513.081755:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879513.081761:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00100000:3.0:1788879513.081763:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879513/real 1788879513] req@ffff94e5045e4700 x1875765290170752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788879513.081768:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879513.081795:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879513.081798:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:3.0:1788879513.081803:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1788879513.081806:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000400:00000001:3.0:1788879513.081807:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879513.081815:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.081815:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:2.0:1788879513.081820:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879513.081822:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:2.0:1788879513.081826:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879513.081826:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.081832:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1788879513.081834:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.081836:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879513.081839:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879513.081844:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1788879513.081845:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290170752/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1788879513.081852:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879513.081859:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.081861:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.081865:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788879513.081867:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879513.081869:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879513.081871:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.081872:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879513.081875:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879513.081878:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879513.081879:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1788879513.081883:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879513.081884:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788879513.081890:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879513.081890:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879513.081897:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879513.081900:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788879513.081902:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879513.081905:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290170752:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879513.081911:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879513.081914:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290171008/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879513.081914:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879513.081927:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879513.081931:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879513.081935:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.081938:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00080000:2.0:1788879513.081941:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1788879513.081942:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879513.081947:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879513.081950:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879513.081951:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879513.081955:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879513.081957:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879513.081960:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879513.081961:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.081965:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.081988:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879513.081990:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.081992:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879513.081994:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879513.081996:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879513.081997:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879513.081999:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879513.082002:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879513.082004:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879513.082006:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879513.082013:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:3.0:1788879513.082014:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290170880:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879513.082018:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788879513.082020:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1788879513.082024:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1788879513.082024:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879513.082028:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879513.082033:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879513.082034:0:80852:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:3.0:1788879513.082036:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879513.082044:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10451 00000100:00000001:3.0:1788879513.082044:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879513.082047:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00080000:2.0:1788879513.082054:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879513.082054:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879513.082060:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879513.082063:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879513.082074:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879513.082076:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879513.082081:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000800:00000100:3.0:1788879513.082081:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:2.0:1788879513.082086:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879513.082088:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879513.082089:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879513.082092:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879513.082093:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788879513.082098:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879513.082101:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879513.082102:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.082106:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:2.0:1788879513.082107:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879513.082112:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879513.082113:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879513.082118:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.082119:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305124364 00000100:00000001:2.0:1788879513.082120:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879513.082125:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879513.082126:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305124905 00000008:00000001:2.0:1788879513.082129:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.082131:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305124364 00000100:00000001:2.0:1788879513.082133:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.082137:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305124905 00000100:00000001:3.0:1788879513.082141:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879513.082159:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879513.082161:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879513.082162:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.082166:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879513.082167:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879513.082171:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879513.082176:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879513.082176:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879513/real 1788879513] req@ffff94e5045e7480 x1875765290170880/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879513.082186:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879513.082191:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879513.082193:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:2.0:1788879513.082197:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290171136/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879513.082198:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879513.082202:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879513.082204:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082228:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879513.082232:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879513.082240:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:3.0:1788879513.082247:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290170880/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879513.082248:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879513.082258:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.082268:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082272:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879513.082275:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082277:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879513.082278:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082280:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879513.082319:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879513.082322:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879513.082326:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879513.082328:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879513.082331:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290170880:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879513.082336:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879513.082338:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879513.082339:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879513.082342:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879513.082364:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879513.082367:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879513.082370:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290171008:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879513.082374:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879513.082376:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879513.082378:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879513.082380:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879513.082384:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.082390:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879513.082392:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074559872 : -117763634991744 : ffff94e5045e6d80) 00000100:00000001:3.0:1788879513.082396:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879513.082409:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879513.082414:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879513.082417:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879513.082418:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.082420:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879513.082422:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.082426:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305124362 00000100:00000001:3.0:1788879513.082428:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305124905 00000100:00000001:3.0:1788879513.082431:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305124362 00000100:00000001:3.0:1788879513.082433:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305124905 00000100:00000001:3.0:1788879513.082435:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879513.082437:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.082439:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879513.082443:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879513/real 1788879513] req@ffff94e5045e6d80 x1875765290171008/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879513.082451:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879513.082452:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879513.082454:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879513.082456:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879513.082457:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082459:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879513.082460:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879513.082464:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6d80 x1875765290171008/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879513.082471:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082473:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879513.082476:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082478:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879513.082479:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082481:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879513.082498:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879513.082501:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879513.082504:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879513.082506:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879513.082510:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290171008:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879513.082513:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082515:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879513.082516:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879513.082518:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879513.082520:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879513.082521:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879513.082523:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879513.082525:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879513.082526:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879513.082528:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879513.082530:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082531:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879513.082533:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879513.082534:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879513.082536:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879513.082537:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879513.082538:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879513.082540:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879513.082541:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879513.082542:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879513.082544:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082545:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082548:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879513.082550:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.082552:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879513.082553:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879513.082554:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879513.082555:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879513.082557:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879513.082560:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879513.082561:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879513.082564:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879513.082566:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290171136:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879513.082569:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879513.082571:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879513.082572:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879513.082574:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879513.082576:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.082580:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879513.082582:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:3.0:1788879513.082585:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879513.082595:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879513.082598:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879513.082600:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879513.082602:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.082604:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879513.082607:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.082610:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305124362 00000100:00000001:3.0:1788879513.082612:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305124905 00000100:00000001:3.0:1788879513.082615:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305124362 00000100:00000001:3.0:1788879513.082617:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305124905 00000100:00000001:3.0:1788879513.082619:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879513.082621:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.082623:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879513.082627:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879513/real 1788879513] req@ffff94e5045e4e00 x1875765290171136/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879513.082650:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879513.082651:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879513.082653:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879513.082654:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879513.082655:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082657:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879513.082658:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879513.082661:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290171136/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879513.082667:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082668:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879513.082670:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082671:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879513.082672:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082673:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879513.082687:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879513.082689:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879513.082691:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879513.082692:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879513.082695:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290171136:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879513.082697:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082699:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879513.082700:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879513.082701:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879513.082702:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879513.082703:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879513.082704:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879513.082706:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879513.082708:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879513.082709:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879513.082710:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082712:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879513.082714:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879513.082715:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879513.082717:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879513.082718:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879513.144971:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879513.144977:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879513.144979:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879513.144984:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879513.144987:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879513.144991:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879513.144994:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879513.144997:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879513.144999:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879513.848979:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879513.848984:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879513.848988:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879513.848989:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879513.849036:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879513.849038:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879513.849041:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879514.168980:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879514.168988:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879514.168991:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879514.168998:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879514.169003:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879514.169008:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879514.169012:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879514.169015:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879514.169019:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879514.873020:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879514.873026:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879514.873031:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879514.873033:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879514.873036:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879514.873038:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879514.873041:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879514.873042:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879514.873044:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879514.873046:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879514.873071:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879514.873073:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879514.873076:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879515.193052:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879515.193059:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879515.193062:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879515.193068:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879515.193072:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879515.193076:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879515.193079:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879515.193082:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879515.193084:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879515.897015:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879515.897023:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879515.897031:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879515.897033:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879515.897113:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879515.897119:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879515.897124:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879516.217008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879516.217017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879516.217019:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879516.217026:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879516.217030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879516.217036:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879516.217039:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879516.217043:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879516.217046:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879516.920987:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879516.920994:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879516.921000:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879516.921001:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879516.921046:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879516.921050:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879516.921054:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879517.241023:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879517.241029:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879517.241031:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879517.241035:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879517.241038:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879517.241041:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879517.241043:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879517.241045:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879517.241047:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879517.944973:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879517.944979:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879517.944981:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879517.944984:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879517.944988:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879517.944990:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879517.944991:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879517.944993:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879517.944996:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879517.944997:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879517.945033:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879517.945035:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879517.945037:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879518.201161:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879518.201177:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879518.201184:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879518.201191:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879518.201199:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879518.201212:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10457 00000100:00080000:0.0:1788879518.201230:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879518.201236:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879518.201240:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879518.201256:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879518.201261:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879518.201267:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879518.201272:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879518.201277:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879518.201280:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879518.201289:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879518.201294:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879518.201299:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879518.201305:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879518.201334:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.201340:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879518.201346:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879518.201363:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879518.201367:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.201374:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879518.201379:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879518.201388:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.201408:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879518.201429:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290171264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879518.201449:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879518.201460:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.201491:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879518.201500:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879518.201507:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879518.201511:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879518.201515:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879518.201523:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879518.201532:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10457 00000100:00080000:0.0:1788879518.201542:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879518.201547:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879518.201551:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879518.201560:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879518.201564:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879518.201569:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879518.201574:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879518.201578:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879518.201583:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879518.201592:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879518.201596:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879518.201600:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879518.201608:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879518.201612:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.201616:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879518.201622:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879518.201627:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.201631:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879518.201638:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879518.201641:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.201644:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879518.201647:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879518.201650:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.201652:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879518.201661:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.201662:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879518.201668:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879518.201676:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:0.0:1788879518.201677:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290171392/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879518.201682:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879518.201692:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879518.201699:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879518.201701:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788879518.201708:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.201710:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.201723:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879518.201729:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290171264:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1788879518.201731:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879518.201748:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1788879518.201751:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879518.201759:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:3.0:1788879518.201760:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1788879518.201764:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1788879518.201768:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.201776:0:80852:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:3.0:1788879518.201778:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1788879518.201792:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10457 02000000:00000001:3.0:1788879518.201793:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879518.201808:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879518.201814:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879518.201817:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879518.201823:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879518.201826:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879518.201830:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879518.201835:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879518.201839:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879518.201842:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879518.201850:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879518.201855:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879518.201859:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879518.201866:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879518.201869:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.201872:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879518.201876:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879518.201879:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.201882:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879518.201889:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879518.201892:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.201894:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879518.201896:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879518.201900:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.201989:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879518.202002:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290171520/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879518.202021:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879518.202026:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.202032:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879518.202038:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879518.202045:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879518.202050:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879518.202054:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879518.202061:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879518.202069:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10457 00000100:00000001:3.0:1788879518.202070:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879518.202076:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00080000:0.0:1788879518.202086:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879518.202087:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879518.202097:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879518.202102:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879518.202111:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879518.202114:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879518.202119:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879518.202125:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879518.202130:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879518.202134:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879518.202145:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879518.202152:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879518.202156:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879518.202164:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879518.202170:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.202174:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879518.202179:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879518.202184:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.202188:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879518.202197:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879518.202199:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.202203:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879518.202205:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879518.202208:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.202216:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879518.202225:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290171648/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879518.202236:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879518.202240:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879518.202244:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788879518.202266:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879518.202296:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879518.202304:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879518.202330:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.202334:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879518.202362:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.202368:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305130026 00000100:00000001:3.0:1788879518.202381:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879518.202389:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305130026 00000100:00000001:3.0:1788879518.202394:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879518.202398:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879518.202402:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.202405:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879518.202414:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879518/real 1788879518] req@ffff94e508978e00 x1875765290171264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879518.202430:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879518.202433:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879518.202438:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879518.202441:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879518.202444:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.202448:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879518.202451:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879518.202457:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290171264/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879518.202478:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.202482:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879518.202486:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.202488:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879518.202489:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.202492:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879518.202495:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879518.202498:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879518.202503:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879518.202506:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879518.202510:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290171264:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879518.202517:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.202520:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879518.202522:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879518.202525:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879518.202529:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879518.202531:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879518.202534:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879518.202539:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879518.202541:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879518.202556:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879518.202558:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.202561:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.202570:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879518.202573:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.202575:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879518.202577:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879518.202579:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879518.202581:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879518.202584:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879518.202587:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879518.202590:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879518.202593:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879518.202597:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290171392:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879518.202601:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879518.202604:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879518.202606:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879518.202608:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879518.202612:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.202619:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879518.202621:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:3.0:1788879518.202638:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879518.202657:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879518.202663:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879518.202666:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879518.202668:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.202670:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879518.202673:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.202678:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305129484 00000100:00000001:3.0:1788879518.202682:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305130026 00000100:00000001:3.0:1788879518.202685:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305129484 00000100:00000001:3.0:1788879518.202688:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305130026 00000100:00000001:3.0:1788879518.202691:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879518.202693:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.202695:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879518.202700:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879518/real 1788879518] req@ffff94e50897a300 x1875765290171392/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879518.202711:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879518.202713:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879518.202718:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879518.202720:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879518.202723:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.202727:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879518.202730:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879518.202738:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290171392/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879518.202753:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.202757:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879518.202762:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.202765:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879518.202768:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.202772:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879518.202797:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879518.202802:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879518.202810:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879518.202824:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879518.202829:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290171392:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879518.202834:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879518.202836:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879518.202838:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879518.202841:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879518.202844:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879518.202847:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879518.202850:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290171520:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879518.202855:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879518.202857:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879518.202859:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879518.202862:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879518.202866:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.202874:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879518.202876:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:3.0:1788879518.202881:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879518.202899:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879518.202941:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879518.202948:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879518.202952:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.202957:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879518.202963:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.202970:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305129483 00000100:00000001:3.0:1788879518.202976:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305130026 00000100:00000001:3.0:1788879518.202981:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305129483 00000100:00000001:3.0:1788879518.202986:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305130026 00000100:00000001:3.0:1788879518.202990:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879518.202994:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.202998:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879518.203007:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879518/real 1788879518] req@ffff94e508979c00 x1875765290171520/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879518.203026:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879518.203029:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879518.203034:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879518.203037:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879518.203040:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.203044:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879518.203046:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879518.203051:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290171520/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879518.203069:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.203071:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879518.203075:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.203077:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879518.203078:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.203080:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879518.203098:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879518.203101:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879518.203104:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879518.203107:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879518.203112:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290171520:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879518.203116:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879518.203118:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879518.203120:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879518.203123:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879518.203125:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879518.203127:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879518.203131:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290171648:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879518.203135:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879518.203138:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879518.203140:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879518.203142:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879518.203146:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.203161:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879518.203163:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:3.0:1788879518.203167:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879518.203185:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879518.203193:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879518.203198:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879518.203201:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.203204:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879518.203209:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.203216:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305129483 00000100:00000001:3.0:1788879518.203221:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305130026 00000100:00000001:3.0:1788879518.203225:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305129483 00000100:00000001:3.0:1788879518.203229:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305130026 00000100:00000001:3.0:1788879518.203234:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879518.203238:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.203241:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879518.203250:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879518/real 1788879518] req@ffff94e50897bb80 x1875765290171648/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879518.203286:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879518.203289:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879518.203295:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879518.203297:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879518.203299:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.203302:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879518.203305:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879518.203340:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290171648/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879518.203366:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.203370:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879518.203374:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.203378:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879518.203380:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.203384:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879518.203406:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879518.203412:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879518.203418:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879518.203423:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879518.203429:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290171648:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879518.203436:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.203441:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879518.203443:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879518.203447:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879518.203452:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879518.203455:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879518.203458:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879518.203462:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879518.203465:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879518.203470:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879518.203473:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.203477:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879518.203480:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879518.203482:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879518.203486:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879518.203489:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879518.203491:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879518.203495:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879518.203498:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879518.203501:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879518.203503:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.203506:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879518.203509:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879518.203511:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879518.203514:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879518.203516:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879518.203518:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879518.203522:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879518.203524:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879518.203528:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879518.203531:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.203535:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879518.203543:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879518.203546:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879518.203550:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879518.203553:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879518.265026:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879518.265038:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879518.265044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879518.265054:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879518.265060:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879518.265066:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879518.265069:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879518.265073:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879518.265076:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879518.969046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879518.969054:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879518.969061:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879518.969063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879518.969107:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879518.969109:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879518.969113:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879519.289000:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879519.289007:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879519.289009:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879519.289013:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879519.289016:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879519.289020:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879519.289022:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879519.289024:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879519.289026:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879519.993065:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879519.993074:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879519.993080:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879519.993082:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879519.993149:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879519.993152:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879519.993156:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879520.313046:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879520.313056:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879520.313059:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879520.313067:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879520.313072:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879520.313078:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879520.313082:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879520.313085:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879520.313088:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879521.017067:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879521.017078:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879521.017084:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879521.017086:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879521.017119:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879521.017123:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879521.017127:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879521.337042:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879521.337071:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879521.337074:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879521.337082:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879521.337087:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879521.337092:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879521.337096:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879521.337112:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879521.337117:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879522.041138:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879522.041145:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879522.041151:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879522.041153:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879522.041219:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879522.041222:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879522.041227:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879522.361070:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879522.361091:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879522.361094:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879522.361102:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879522.361110:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879522.361116:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879522.361120:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879522.361124:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879522.361128:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879523.065072:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879523.065081:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879523.065087:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879523.065088:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879523.065138:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879523.065141:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879523.065144:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879523.321099:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879523.321112:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879523.321116:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879523.321121:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879523.321127:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879523.321137:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10462 00000100:00080000:0.0:1788879523.321164:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879523.321167:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879523.321169:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879523.321181:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879523.321185:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879523.321189:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879523.321193:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879523.321195:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879523.321198:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879523.321204:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879523.321208:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879523.321212:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879523.321216:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879523.321220:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321224:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879523.321227:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879523.321244:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879523.321247:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321252:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879523.321256:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879523.321263:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321313:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879523.321328:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290171776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879523.321361:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879523.321370:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321395:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879523.321399:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879523.321403:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879523.321404:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879523.321406:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879523.321409:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879523.321412:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10462 00000100:00080000:0.0:1788879523.321417:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879523.321421:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879523.321424:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879523.321428:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879523.321430:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879523.321433:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879523.321436:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879523.321439:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879523.321441:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879523.321444:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879523.321446:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879523.321448:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879523.321461:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879523.321461:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788879523.321470:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321473:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879523.321478:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879523.321483:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.321484:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879523.321486:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.321490:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1788879523.321494:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879523.321495:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788879523.321500:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.321503:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321505:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879523.321509:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879523.321517:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321536:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879523.321538:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879523.321544:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788879523.321547:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979180 x1875765290171904/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879523.321551:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321573:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:3.0:1788879523.321576:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290171776:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879523.321581:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.321597:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788879523.321598:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879523.321605:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1788879523.321606:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1788879523.321620:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:3.0:1788879523.321621:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321633:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:3.0:1788879523.321634:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1788879523.321646:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879523.321652:0:80852:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:3.0:1788879523.321655:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879523.321657:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10462 00000100:00080000:0.0:1788879523.321670:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879523.321672:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879523.321673:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879523.321679:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879523.321680:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879523.321682:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879523.321685:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879523.321687:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879523.321689:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879523.321704:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 00000100:00000001:0.0:1788879523.321706:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879523.321711:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1788879523.321716:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879523.321720:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879523.321722:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879523.321726:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879523.321728:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321730:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879523.321732:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879523.321734:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321736:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879523.321740:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879523.321741:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321743:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879523.321745:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879523.321748:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321753:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879523.321777:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290172032/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788879523.321784:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879523.321802:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879523.321805:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321809:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.321809:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:0.0:1788879523.321817:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879523.321821:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1788879523.321823:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879523.321826:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321829:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788879523.321838:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:0.0:1788879523.321840:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879523.321849:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879523.321852:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879523.321857:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10462 00000100:00000001:3.0:1788879523.321866:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305135145 00000100:00080000:0.0:1788879523.321869:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879523.321894:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879523.321895:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879523.321899:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879523.321900:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879523.321902:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879523.321904:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879523.321906:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879523.321907:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879523.321911:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879523.321914:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879523.321918:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879523.321922:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879523.321925:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321927:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879523.321929:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879523.321932:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321934:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879523.321940:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879523.321943:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879523.321945:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879523.321946:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879523.321948:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.321951:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879523.321964:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305135146 00000100:00000001:3.0:1788879523.321973:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879523.321978:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879523.321981:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.321986:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879523.321997:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879523.321998:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879523/real 1788879523] req@ffff94e508978380 x1875765290171776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879523.322031:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290172160/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788879523.322034:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879523.322041:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879523.322054:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879523.322056:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879523.322069:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.322069:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879523.322081:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879523.322085:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.322088:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879523.322094:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879523.322104:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290171776/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879523.322117:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322123:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879523.322129:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322131:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879523.322133:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322136:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879523.322139:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879523.322144:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879523.322168:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879523.322172:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879523.322176:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290171776:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879523.322182:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322185:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879523.322186:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879523.322189:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879523.322193:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879523.322196:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879523.322199:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879523.322204:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879523.322206:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879523.322209:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879523.322210:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322213:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322221:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879523.322224:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.322227:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879523.322229:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879523.322231:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879523.322233:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879523.322235:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879523.322238:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879523.322240:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879523.322254:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879523.322257:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290171904:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879523.322261:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879523.322264:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879523.322266:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879523.322268:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879523.322302:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.322324:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879523.322325:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 00000100:00000001:3.0:1788879523.322330:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879523.322350:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879523.322355:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879523.322358:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879523.322360:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.322362:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879523.322365:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.322369:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305134604 00000100:00000001:3.0:1788879523.322373:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305135145 00000100:00000001:3.0:1788879523.322376:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305134604 00000100:00000001:3.0:1788879523.322378:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305135146 00000100:00000001:3.0:1788879523.322381:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879523.322383:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.322385:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879523.322389:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879523/real 1788879523] req@ffff94e508979180 x1875765290171904/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879523.322398:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879523.322399:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879523.322402:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879523.322404:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879523.322405:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322407:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879523.322409:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879523.322413:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290171904/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879523.322421:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322423:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879523.322425:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322427:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879523.322428:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322430:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879523.322448:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879523.322451:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879523.322455:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879523.322457:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879523.322461:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290171904:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879523.322466:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879523.322467:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879523.322469:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879523.322471:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879523.322473:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879523.322475:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879523.322478:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290172032:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879523.322482:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879523.322484:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879523.322485:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879523.322487:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879523.322491:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.322496:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879523.322497:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000100:00000001:3.0:1788879523.322501:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879523.322530:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879523.322535:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879523.322538:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879523.322540:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.322542:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879523.322545:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.322550:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305134602 00000100:00000001:3.0:1788879523.322554:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305135145 00000100:00000001:3.0:1788879523.322556:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305134602 00000100:00000001:3.0:1788879523.322559:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305135146 00000100:00000001:3.0:1788879523.322562:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879523.322565:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.322567:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879523.322572:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879523/real 1788879523] req@ffff94e50897b800 x1875765290172032/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879523.322582:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879523.322584:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879523.322587:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879523.322589:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879523.322590:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322592:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879523.322594:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879523.322599:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290172032/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879523.322607:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322610:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879523.322612:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322614:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879523.322616:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322618:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879523.322635:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879523.322638:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879523.322641:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879523.322643:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879523.322647:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290172032:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879523.322651:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879523.322653:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879523.322655:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879523.322658:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879523.322660:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879523.322662:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879523.322666:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290172160:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879523.322670:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879523.322673:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879523.322675:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879523.322678:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879523.322681:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.322686:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879523.322688:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:3.0:1788879523.322692:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879523.322714:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879523.322718:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879523.322721:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879523.322722:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.322725:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879523.322727:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.322731:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305134602 00000100:00000001:3.0:1788879523.322734:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305135145 00000100:00000001:3.0:1788879523.322737:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305134602 00000100:00000001:3.0:1788879523.322740:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305135146 00000100:00000001:3.0:1788879523.322742:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879523.322745:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.322747:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879523.322751:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879523/real 1788879523] req@ffff94e50897bb80 x1875765290172160/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879523.322760:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879523.322761:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879523.322769:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879523.322771:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879523.322772:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322774:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879523.322776:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879523.322780:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290172160/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879523.322788:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322791:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879523.322794:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322796:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879523.322798:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322800:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879523.322817:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879523.322820:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879523.322824:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879523.322836:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879523.322840:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290172160:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879523.322844:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322846:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879523.322848:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879523.322850:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879523.322852:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879523.322854:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879523.322855:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879523.322859:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879523.322860:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879523.322864:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879523.322865:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322867:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879523.322869:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879523.322870:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879523.322872:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879523.322874:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879523.322875:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879523.322878:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879523.322879:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879523.322882:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879523.322884:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322886:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879523.322887:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879523.322888:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879523.322890:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879523.322891:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879523.322893:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879523.322894:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879523.322896:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879523.322898:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879523.322900:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322902:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879523.322906:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879523.322908:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879523.322911:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879523.322913:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879523.385079:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879523.385087:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879523.385089:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879523.385095:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879523.385099:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879523.385103:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879523.385106:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879523.385109:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879523.385112:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879524.089053:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879524.089058:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879524.089062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879524.089083:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879524.089086:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879524.089092:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879524.089096:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879524.089100:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879524.089103:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879524.089107:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879524.089154:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879524.089159:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879524.089163:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879524.409028:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879524.409037:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879524.409040:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879524.409047:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879524.409051:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879524.409056:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879524.409059:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879524.409063:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879524.409066:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879525.113025:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879525.113032:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879525.113039:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879525.113041:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879525.113136:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879525.113139:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879525.113143:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879525.433006:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879525.433013:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879525.433017:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879525.433023:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879525.433027:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879525.433033:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879525.433036:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879525.433039:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879525.433043:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879526.136991:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879526.136996:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879526.137001:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879526.137002:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879526.137024:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879526.137027:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879526.137031:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879526.457037:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879526.457049:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879526.457054:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879526.457063:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879526.457070:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879526.457076:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879526.457082:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879526.457104:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879526.457109:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879527.161019:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879527.161027:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879527.161030:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879527.161034:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879527.161040:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879527.161042:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879527.161044:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879527.161046:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879527.161068:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879527.161070:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879527.161097:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879527.161101:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879527.161104:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879527.481003:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879527.481012:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879527.481015:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879527.481021:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879527.481026:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879527.481031:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879527.481034:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879527.481038:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879527.481041:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879528.185023:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879528.185030:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879528.185036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879528.185038:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879528.185064:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879528.185067:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879528.185070:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879528.441086:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879528.441099:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879528.441103:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879528.441109:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879528.441115:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879528.441125:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10467 00000100:00080000:2.0:1788879528.441139:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879528.441144:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879528.441147:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879528.441157:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879528.441160:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879528.441165:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879528.441168:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879528.441171:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879528.441174:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879528.441182:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879528.441188:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879528.441194:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879528.441200:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879528.441207:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879528.441212:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879528.441216:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879528.441261:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879528.441265:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879528.441271:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879528.441277:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879528.441285:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879528.441307:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879528.441338:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290172288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879528.441360:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879528.441369:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879528.441394:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879528.441401:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879528.441411:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879528.441413:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879528.441415:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879528.441420:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879528.441424:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10467 00000100:00080000:2.0:1788879528.441431:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879528.441434:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879528.441437:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879528.441443:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879528.441446:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879528.441449:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879528.441452:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879528.441455:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879528.441457:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879528.441461:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788879528.441463:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879528.441471:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.441472:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788879528.441478:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879528.441481:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879528.441486:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788879528.441489:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879528.441490:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879528.441502:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879528.441504:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879528.441507:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788879528.441511:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:2.0:1788879528.441515:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788879528.441518:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788879528.441522:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879528.441528:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879528.441530:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290172288:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879528.441537:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879528.441541:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879528.441542:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879528.441548:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879528.441549:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879528.441551:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879528.441555:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879528.441558:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879528.441569:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879528.441575:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1788879528.441578:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4000 x1875765290172416/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879528.441586:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879528.441606:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879528.441610:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.441615:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879528.441616:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.441622:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00080000:2.0:1788879528.441624:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879528.441633:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788879528.441634:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879528.441640:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879528.441643:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879528.441648:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879528.441651:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10467 00000100:00080000:2.0:1788879528.441669:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879528.441673:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879528.441675:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879528.441679:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879528.441681:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879528.441684:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879528.441691:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000800:00000100:3.0:1788879528.441692:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879528.441698:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879528.441703:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879528.441711:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879528.441714:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879528.441716:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879528.441722:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879528.441725:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788879528.441727:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879528.441732:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879528.441734:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.441740:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879528.441742:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879528.441748:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879528.441751:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788879528.441753:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879528.441761:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.441762:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305140265 02000000:00000001:2.0:1788879528.441770:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879528.441774:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879528.441778:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879528.441779:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1788879528.441780:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879528.441787:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.441795:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305140265 00000100:00000001:2.0:1788879528.441799:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.441804:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00100000:2.0:1788879528.441812:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290172544/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879528.441814:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879528.441825:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.441835:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879528.441842:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879528.441848:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879528.441851:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879528/real 1788879528] req@ffff94e5045e7b80 x1875765290172288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879528.441853:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879528.441862:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879528.441878:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879528.441885:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:3.0:1788879528.441886:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1788879528.441890:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:3.0:1788879528.441891:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879528.441898:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879528.441959:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879528.441963:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879528.441967:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879528.441968:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879528.441976:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879528.441982:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10467 00000100:00100000:3.0:1788879528.441990:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290172288/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879528.441995:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879528.442005:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879528.442011:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879528.442016:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788879528.442019:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879528.442023:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1788879528.442025:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879528.442032:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879528.442033:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788879528.442040:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879528.442041:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879528.442047:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879528.442049:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879528.442053:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879528.442054:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879528.442059:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879528.442066:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00080000:3.0:1788879528.442069:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788879528.442072:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879528.442078:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879528.442083:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879528.442086:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879528.442089:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00080000:3.0:1788879528.442094:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000008:00100000:2.0:1788879528.442096:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879528.442103:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000001:2.0:1788879528.442104:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879528.442115:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879528.442118:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290172288:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879528.442124:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879528.442127:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879528.442129:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442133:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879528.442138:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879528.442141:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879528.442146:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879528.442148:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:2.0:1788879528.442151:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:3.0:1788879528.442152:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879528.442159:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879528.442162:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879528.442165:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879528.442168:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879528.442174:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879528.442178:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442182:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879528.442183:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442191:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1788879528.442195:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290172672/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879528.442197:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442208:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879528.442212:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879528.442216:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879528.442221:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879528.442227:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879528.442228:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788879528.442230:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879528.442256:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879528.442266:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879528.442271:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879528.442280:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879528.442289:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290172416:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879528.442297:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879528.442302:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879528.442305:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879528.442309:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879528.442314:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442324:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879528.442325:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074548224 : -117763635003392 : ffff94e5045e4000) 00000100:00000001:3.0:1788879528.442331:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879528.442352:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879528.442358:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879528.442362:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879528.442364:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442366:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879528.442370:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442375:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305139724 00000100:00000001:3.0:1788879528.442379:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305140265 00000100:00000001:3.0:1788879528.442382:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305139724 00000100:00000001:3.0:1788879528.442385:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305140265 00000100:00000001:3.0:1788879528.442388:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879528.442390:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442394:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879528.442400:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879528/real 1788879528] req@ffff94e5045e4000 x1875765290172416/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879528.442411:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879528.442413:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879528.442416:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879528.442419:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879528.442421:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442423:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879528.442425:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879528.442429:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4000 x1875765290172416/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879528.442438:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442441:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879528.442444:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442446:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879528.442447:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442449:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879528.442469:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879528.442472:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879528.442476:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879528.442479:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879528.442483:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290172416:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879528.442487:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879528.442489:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879528.442491:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879528.442494:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879528.442496:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879528.442499:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879528.442503:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290172544:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879528.442510:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879528.442512:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879528.442514:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879528.442516:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879528.442520:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442525:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879528.442527:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:3.0:1788879528.442532:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879528.442543:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879528.442548:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879528.442551:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879528.442552:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442555:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879528.442557:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442561:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305139722 00000100:00000001:3.0:1788879528.442565:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305140265 00000100:00000001:3.0:1788879528.442567:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305139722 00000100:00000001:3.0:1788879528.442570:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305140265 00000100:00000001:3.0:1788879528.442573:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879528.442575:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442577:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879528.442581:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879528/real 1788879528] req@ffff94e5045e4e00 x1875765290172544/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879528.442591:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879528.442592:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879528.442594:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879528.442596:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879528.442598:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442600:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879528.442601:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879528.442606:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290172544/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879528.442622:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442625:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879528.442628:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442629:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879528.442631:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442633:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879528.442651:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879528.442653:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879528.442657:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879528.442659:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879528.442663:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290172544:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879528.442667:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442670:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879528.442672:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879528.442674:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879528.442676:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879528.442677:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879528.442679:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879528.442682:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879528.442684:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879528.442686:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879528.442688:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442690:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879528.442692:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879528.442693:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879528.442695:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879528.442697:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879528.442698:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879528.442700:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879528.442701:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879528.442703:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879528.442705:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442707:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442711:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879528.442714:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442716:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879528.442718:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879528.442720:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879528.442722:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879528.442724:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879528.442726:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879528.442728:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879528.442730:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879528.442734:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290172672:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879528.442738:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879528.442740:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879528.442742:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879528.442744:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879528.442747:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442752:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879528.442753:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074559872 : -117763634991744 : ffff94e5045e6d80) 00000100:00000001:3.0:1788879528.442757:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879528.442769:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879528.442773:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879528.442776:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879528.442778:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442780:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879528.442783:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442786:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305139722 00000100:00000001:3.0:1788879528.442790:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305140265 00000100:00000001:3.0:1788879528.442792:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305139722 00000100:00000001:3.0:1788879528.442795:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305140265 00000100:00000001:3.0:1788879528.442798:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879528.442800:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442802:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879528.442807:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879528/real 1788879528] req@ffff94e5045e6d80 x1875765290172672/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879528.442824:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879528.442826:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879528.442829:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879528.442831:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879528.442833:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442835:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879528.442836:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879528.442840:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6d80 x1875765290172672/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879528.442848:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442851:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879528.442853:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442855:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879528.442857:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442859:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879528.442877:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879528.442880:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879528.442883:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879528.442886:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879528.442890:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290172672:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879528.442894:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442896:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879528.442898:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879528.442899:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879528.442929:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879528.442932:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879528.442933:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879528.442936:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879528.442938:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879528.442941:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879528.442943:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442946:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879528.442949:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879528.442951:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879528.442953:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879528.442955:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879528.505083:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879528.505090:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879528.505093:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879528.505099:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879528.505104:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879528.505109:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879528.505112:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879528.505116:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879528.505119:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879529.209040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879529.209048:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879529.209055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879529.209057:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879529.209082:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879529.209086:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879529.209090:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879529.528980:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879529.528988:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879529.528992:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879529.528998:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879529.529003:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879529.529008:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879529.529011:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879529.529015:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879529.529018:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879530.233091:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879530.233113:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879530.233121:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879530.233124:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879530.233215:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879530.233221:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879530.233246:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879530.553015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879530.553024:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879530.553044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879530.553050:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879530.553055:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879530.553060:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879530.553065:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879530.553069:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879530.553073:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879531.256989:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879531.256995:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879531.257001:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879531.257002:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879531.257023:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879531.257026:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879531.257029:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879531.577040:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879531.577049:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879531.577053:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879531.577062:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879531.577069:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879531.577076:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879531.577082:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879531.577087:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879531.577092:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879532.281030:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879532.281036:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879532.281042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879532.281043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879532.281094:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879532.281110:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879532.281116:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879532.600971:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879532.600990:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879532.600993:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879532.600998:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879532.601003:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879532.601007:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879532.601010:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879532.601013:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879532.601016:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879533.305017:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879533.305037:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879533.305042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879533.305044:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879533.305070:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879533.305073:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879533.305076:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879533.560969:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879533.560980:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879533.560983:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879533.560986:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879533.560991:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879533.560999:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10472 00000100:00080000:2.0:1788879533.561010:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879533.561013:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879533.561014:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879533.561023:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879533.561025:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879533.561028:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879533.561030:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879533.561031:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879533.561033:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879533.561036:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879533.561038:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879533.561040:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879533.561044:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879533.561046:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879533.561049:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879533.561050:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879533.561063:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879533.561065:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879533.561068:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879533.561070:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879533.561075:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879533.561087:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879533.561096:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290172800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879533.561105:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879533.561109:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879533.561123:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879533.561126:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879533.561128:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879533.561130:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879533.561131:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879533.561133:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879533.561135:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10472 00000100:00080000:2.0:1788879533.561137:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879533.561138:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879533.561139:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879533.561141:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879533.561142:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879533.561143:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879533.561145:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879533.561146:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879533.561148:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879533.561150:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879533.561152:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879533.561154:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879533.561160:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879533.561162:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879533.561164:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879533.561169:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879533.561169:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:2.0:1788879533.561174:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561175:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879533.561176:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561180:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788879533.561181:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879533.561183:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788879533.561184:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879533.561188:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879533.561188:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879533.561191:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879533.561193:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879533.561195:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879533.561198:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879533.561213:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879533.561217:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879533.561220:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290172928/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879533.561224:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290172800:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879533.561233:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788879533.561237:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788879533.561239:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788879533.561240:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879533.561242:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879533.561245:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879533.561245:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1788879533.561250:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1788879533.561253:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879533.561254:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879533.561258:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879533.561259:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879533.561262:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879533.561264:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10472 00000100:00080000:2.0:1788879533.561267:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879533.561269:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879533.561270:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879533.561273:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879533.561273:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1788879533.561274:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879533.561277:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000020:00000001:2.0:1788879533.561278:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879533.561282:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1788879533.561288:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879533.561290:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879533.561292:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879533.561295:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879533.561297:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879533.561298:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879533.561300:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879533.561302:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879533.561303:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879533.561305:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879533.561306:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879533.561308:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879533.561311:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879533.561313:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879533.561314:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879533.561315:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879533.561317:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879533.561323:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879533.561325:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290173056/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788879533.561332:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879533.561335:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879533.561339:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879533.561342:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879533.561345:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879533.561349:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879533.561351:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879533.561352:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879533.561357:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879533.561359:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879533.561361:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879533.561363:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561366:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:2.0:1788879533.561368:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10472 00000100:00000001:3.0:1788879533.561372:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879533.561375:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879533.561378:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305145385 00000100:00000001:2.0:1788879533.561379:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879533.561383:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879533.561387:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879533.561390:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879533.561390:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1788879533.561392:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879533.561396:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879533.561398:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879533.561399:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879533.561402:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879533.561404:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879533.561406:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879533.561406:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305145385 02000000:00000001:2.0:1788879533.561409:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879533.561412:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561412:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879533.561418:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879533.561420:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879533.561420:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1788879533.561426:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561426:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879533.561428:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561432:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1788879533.561434:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879533.561437:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879533.561438:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879533.561441:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:3.0:1788879533.561442:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879533/real 1788879533] req@ffff94e5045e7480 x1875765290172800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879533.561444:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879533.561450:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879533.561458:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290173184/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1788879533.561465:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788879533.561466:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879533.561470:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879533.561470:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879533.561474:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561478:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879533.561483:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879533.561486:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561490:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879533.561492:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879533.561499:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290172800/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879533.561505:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561508:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879533.561511:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561512:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879533.561512:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561514:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879533.561516:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879533.561518:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879533.561521:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879533.561523:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879533.561526:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290172800:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879533.561530:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561532:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879533.561533:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879533.561535:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879533.561538:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879533.561539:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879533.561541:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879533.561545:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879533.561546:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879533.561548:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879533.561549:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561551:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561558:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879533.561560:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561561:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879533.561563:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879533.561564:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879533.561565:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879533.561566:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879533.561567:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879533.561569:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879533.561570:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879533.561572:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290172928:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879533.561575:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879533.561576:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879533.561577:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879533.561578:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879533.561580:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561586:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879533.561587:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:3.0:1788879533.561591:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879533.561601:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879533.561604:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879533.561605:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879533.561606:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561608:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879533.561609:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561612:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305144843 00000100:00000001:3.0:1788879533.561615:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305145385 00000100:00000001:3.0:1788879533.561616:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305144843 00000100:00000001:3.0:1788879533.561618:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305145385 00000100:00000001:3.0:1788879533.561619:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879533.561621:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561622:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879533.561624:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879533/real 1788879533] req@ffff94e5045e4700 x1875765290172928/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879533.561635:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879533.561636:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879533.561637:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879533.561638:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879533.561639:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561640:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879533.561641:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879533.561643:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290172928/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879533.561648:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561649:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879533.561651:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561652:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879533.561653:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561654:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879533.561665:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879533.561666:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879533.561668:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879533.561670:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879533.561672:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290172928:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879533.561674:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879533.561675:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879533.561676:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879533.561678:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879533.561679:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879533.561680:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879533.561682:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290173056:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879533.561684:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879533.561685:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879533.561686:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879533.561687:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879533.561689:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561693:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879533.561694:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:3.0:1788879533.561696:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879533.561702:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879533.561705:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879533.561706:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879533.561707:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561708:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879533.561709:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561711:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305144841 00000100:00000001:3.0:1788879533.561713:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305145385 00000100:00000001:3.0:1788879533.561715:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305144841 00000100:00000001:3.0:1788879533.561716:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305145385 00000100:00000001:3.0:1788879533.561718:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879533.561719:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561720:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879533.561722:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879533/real 1788879533] req@ffff94e5045e6680 x1875765290173056/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879533.561726:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879533.561727:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879533.561728:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879533.561729:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879533.561730:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561731:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879533.561731:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879533.561734:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290173056/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879533.561738:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561739:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879533.561741:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561743:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879533.561743:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561744:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879533.561753:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879533.561754:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879533.561756:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879533.561758:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879533.561760:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290173056:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879533.561762:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879533.561762:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879533.561763:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879533.561765:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879533.561766:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879533.561767:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879533.561768:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290173184:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879533.561770:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879533.561771:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879533.561772:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879533.561773:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879533.561775:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561777:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879533.561778:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:3.0:1788879533.561781:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879533.561786:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879533.561788:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879533.561789:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879533.561790:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561791:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879533.561792:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561794:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305144841 00000100:00000001:3.0:1788879533.561796:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305145385 00000100:00000001:3.0:1788879533.561798:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305144841 00000100:00000001:3.0:1788879533.561799:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305145385 00000100:00000001:3.0:1788879533.561801:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879533.561802:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561803:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879533.561805:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879533/real 1788879533] req@ffff94e5045e6a00 x1875765290173184/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879533.561810:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879533.561810:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879533.561811:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879533.561812:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879533.561813:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561814:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879533.561815:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879533.561817:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290173184/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879533.561821:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561823:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879533.561824:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561825:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879533.561826:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561827:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879533.561837:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879533.561838:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879533.561840:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879533.561841:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879533.561843:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290173184:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879533.561846:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561847:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879533.561848:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879533.561849:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879533.561851:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879533.561852:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879533.561853:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879533.561854:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879533.561855:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879533.561856:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879533.561857:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561858:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879533.561859:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879533.561860:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879533.561861:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879533.561862:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879533.561863:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879533.561864:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879533.561865:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879533.561866:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879533.561867:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561868:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879533.561869:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879533.561869:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879533.561870:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879533.561871:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879533.561872:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879533.561873:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879533.561873:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879533.561874:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879533.561875:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561876:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879533.561878:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879533.561878:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879533.561880:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879533.561881:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879533.624995:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879533.625002:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879533.625004:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879533.625008:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879533.625011:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879533.625014:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879533.625017:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879533.625019:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879533.625021:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879534.329063:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879534.329074:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879534.329080:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879534.329086:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879534.329089:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879534.329094:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879534.329098:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879534.329100:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879534.329103:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879534.329105:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879534.329137:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879534.329141:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879534.329145:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879534.649058:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879534.649065:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879534.649068:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879534.649073:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879534.649079:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879534.649083:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879534.649104:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879534.649107:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879534.649109:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879535.353113:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879535.353122:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879535.353129:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879535.353130:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879535.353157:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879535.353160:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879535.353163:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879535.673019:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879535.673026:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879535.673040:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879535.673045:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879535.673049:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879535.673053:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879535.673055:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879535.673058:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879535.673060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879536.377052:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879536.377061:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879536.377068:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879536.377070:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879536.377100:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879536.377104:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879536.377108:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879536.697020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879536.697029:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879536.697033:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879536.697040:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879536.697044:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879536.697067:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879536.697071:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879536.697074:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879536.697077:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879537.401013:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879537.401026:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879537.401030:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879537.401036:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879537.401044:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879537.401047:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879537.401051:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879537.401056:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879537.401095:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879537.401099:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879537.401139:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879537.401143:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879537.401147:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879537.721039:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879537.721047:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879537.721050:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879537.721056:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879537.721061:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879537.721066:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879537.721069:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879537.721072:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879537.721076:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879538.425022:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879538.425030:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879538.425036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879538.425037:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879538.425061:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879538.425064:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879538.425067:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879538.681116:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879538.681129:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879538.681133:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879538.681138:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879538.681144:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879538.681153:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10477 00000100:00080000:0.0:1788879538.681205:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879538.681210:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879538.681213:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879538.681223:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879538.681227:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879538.681231:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879538.681234:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879538.681237:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879538.681239:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879538.681245:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879538.681248:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879538.681252:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879538.681274:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879538.681278:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879538.681282:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879538.681285:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879538.681300:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879538.681302:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879538.681307:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879538.681310:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879538.681316:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879538.681335:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879538.681353:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290173312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879538.681388:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879538.681398:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879538.681423:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879538.681430:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879538.681436:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879538.681438:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879538.681441:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879538.681447:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879538.681452:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10477 00000100:00080000:0.0:1788879538.681459:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879538.681463:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879538.681466:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879538.681468:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788879538.681472:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879538.681477:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879538.681477:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879538.681485:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788879538.681486:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879538.681492:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1788879538.681494:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879538.681499:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879538.681501:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879538.681508:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:3.0:1788879538.681512:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1788879538.681517:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:3.0:1788879538.681520:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879538.681525:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879538.681528:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879538.681531:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879538.681543:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788879538.681544:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290173312:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788879538.681554:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.681558:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788879538.681560:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788879538.681567:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:0.0:1788879538.681569:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788879538.681573:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788879538.681576:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879538.681580:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879538.681582:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879538.681589:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879538.681611:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879538.681615:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879538.681619:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879538.681622:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879538.681626:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.681632:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879538.681637:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:0.0:1788879538.681640:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879538.681657:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290173440/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879538.681663:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879538.681679:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879538.681685:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879538.681694:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879538.681699:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879538.681706:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879538.681709:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879538.681712:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879538.681717:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000800:00000100:3.0:1788879538.681723:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1788879538.681727:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10477 00000100:00080000:0.0:1788879538.681737:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879538.681740:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879538.681743:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879538.681744:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:0.0:1788879538.681750:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879538.681753:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1788879538.681755:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879538.681760:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879538.681762:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879538.681768:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:0.0:1788879538.681770:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879538.681776:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879538.681777:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879538.681783:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879538.681790:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879538.681792:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879538.681794:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879538.681796:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305150505 02000000:00000001:0.0:1788879538.681799:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879538.681805:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879538.681808:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879538.681812:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879538.681813:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00000001:0.0:1788879538.681820:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.681826:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305150505 00000100:00000001:0.0:1788879538.681828:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.681835:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879538.681842:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879538.681843:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879538.681846:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879538.681851:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.681853:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879538.681862:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1788879538.681865:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879538/real 1788879538] req@ffff94e508979c00 x1875765290173312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788879538.681868:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879538.681876:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879538.681891:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788879538.681893:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879538.681894:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879538.681902:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00100000:0.0:1788879538.681912:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290173568/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879538.681936:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879538.681939:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.681944:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879538.681965:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879538.681975:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290173312/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879538.682013:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879538.682017:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879538.682021:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682025:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879538.682030:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682032:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879538.682039:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879538.682040:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879538.682045:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879538.682051:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879538.682069:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1788879538.682077:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788879538.682078:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879538.682084:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1788879538.682087:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1788879538.682096:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879538.682099:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00080000:0.0:1788879538.682107:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10477 00000100:00000001:3.0:1788879538.682108:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:0.0:1788879538.682118:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1788879538.682120:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290173312:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879538.682127:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879538.682132:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879538.682133:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788879538.682141:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879538.682141:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879538.682144:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788879538.682146:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:3.0:1788879538.682151:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:0.0:1788879538.682152:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:3.0:1788879538.682159:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1788879538.682160:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879538.682189:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:0.0:1788879538.682192:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000020:00000001:3.0:1788879538.682193:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:0.0:1788879538.682199:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:3.0:1788879538.682203:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879538.682226:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:0.0:1788879538.682228:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879538.682233:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1788879538.682236:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682238:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879538.682241:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879538.682247:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879538.682249:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879538.682256:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879538.682259:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879538.682260:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000008:00100000:0.0:1788879538.682262:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879538.682268:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788879538.682270:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879538.682280:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682283:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788879538.682288:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879538.682291:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788879538.682293:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879538.682300:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879538.682300:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1788879538.682308:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879538.682310:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788879538.682312:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682318:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879538.682325:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879538.682329:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879538.682337:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788879538.682342:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290173696/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879538.682346:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879538.682365:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290173440:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879538.682377:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879538.682384:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788879538.682385:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788879538.682387:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879538.682393:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879538.682397:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879538.682400:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879538.682411:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682414:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879538.682420:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:3.0:1788879538.682428:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879538.682453:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879538.682461:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879538.682465:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879538.682467:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682469:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879538.682472:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682478:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305149964 00000100:00000001:3.0:1788879538.682482:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305150505 00000100:00000001:3.0:1788879538.682485:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305149964 00000100:00000001:3.0:1788879538.682488:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305150505 00000100:00000001:3.0:1788879538.682491:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879538.682494:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682496:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879538.682502:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879538/real 1788879538] req@ffff94e50897a300 x1875765290173440/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879538.682513:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879538.682514:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879538.682518:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879538.682520:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879538.682522:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.682524:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879538.682526:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879538.682531:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290173440/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879538.682548:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.682551:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879538.682554:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.682556:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879538.682558:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.682560:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879538.682580:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879538.682583:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879538.682587:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879538.682589:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879538.682593:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290173440:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879538.682615:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879538.682617:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879538.682619:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879538.682622:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879538.682624:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879538.682627:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879538.682630:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290173568:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879538.682634:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879538.682636:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879538.682638:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879538.682640:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879538.682643:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682650:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879538.682651:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:3.0:1788879538.682655:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879538.682668:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879538.682672:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879538.682675:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879538.682676:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682679:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879538.682682:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682685:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305149962 00000100:00000001:3.0:1788879538.682702:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305150505 00000100:00000001:3.0:1788879538.682705:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305149962 00000100:00000001:3.0:1788879538.682708:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305150505 00000100:00000001:3.0:1788879538.682710:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879538.682717:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682719:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879538.682724:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879538/real 1788879538] req@ffff94e508978e00 x1875765290173568/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879538.682733:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879538.682735:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879538.682738:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879538.682740:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879538.682741:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.682743:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879538.682745:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879538.682749:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290173568/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879538.682757:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.682759:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879538.682762:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.682764:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879538.682765:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.682767:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879538.682785:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879538.682788:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879538.682791:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879538.682793:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879538.682798:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290173568:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879538.682802:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.682805:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879538.682806:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879538.682808:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879538.682811:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879538.682812:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879538.682814:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879538.682817:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879538.682819:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879538.682822:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879538.682824:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.682826:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879538.682827:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879538.682829:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879538.682830:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879538.682832:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879538.682833:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879538.682835:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879538.682837:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879538.682839:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879538.682840:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.682842:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.682846:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879538.682848:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682850:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879538.682851:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879538.682853:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879538.682854:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879538.682856:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879538.682858:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879538.682860:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879538.682862:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879538.682866:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290173696:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879538.682870:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879538.682872:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879538.682874:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879538.682876:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879538.682880:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682885:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879538.682886:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:3.0:1788879538.682891:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879538.682928:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879538.682934:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879538.682937:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879538.682939:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682941:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879538.682962:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682968:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305149963 00000100:00000001:3.0:1788879538.682971:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305150505 00000100:00000001:3.0:1788879538.682974:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305149963 00000100:00000001:3.0:1788879538.682977:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305150505 00000100:00000001:3.0:1788879538.682979:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879538.682982:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.682984:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879538.682990:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879538/real 1788879538] req@ffff94e50897ad80 x1875765290173696/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879538.682999:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879538.683001:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879538.683004:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879538.683006:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879538.683008:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.683010:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879538.683012:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879538.683017:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290173696/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879538.683025:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.683028:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879538.683031:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.683033:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879538.683035:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.683038:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879538.683072:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879538.683076:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879538.683079:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879538.683082:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879538.683086:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290173696:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879538.683090:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.683092:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879538.683094:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879538.683095:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879538.683098:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879538.683099:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879538.683100:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879538.683103:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879538.683104:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879538.683108:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879538.683109:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.683112:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879538.683115:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879538.683117:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879538.683119:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879538.683121:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879538.745016:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879538.745023:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879538.745026:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879538.745031:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879538.745035:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879538.745039:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879538.745042:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879538.745045:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879538.745047:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879539.449056:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879539.449063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879539.449070:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879539.449071:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879539.449098:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879539.449101:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879539.449105:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879539.769014:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879539.769024:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879539.769027:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879539.769034:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879539.769039:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879539.769044:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879539.769048:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879539.769051:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879539.769055:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879540.473045:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879540.473054:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879540.473063:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879540.473065:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879540.473095:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879540.473098:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879540.473101:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879540.792998:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879540.793004:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879540.793023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879540.793029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879540.793034:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879540.793039:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879540.793043:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879540.793046:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879540.793049:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879541.497068:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879541.497076:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879541.497084:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879541.497086:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879541.497125:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879541.497128:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879541.497132:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879541.816980:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879541.816988:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879541.816991:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879541.816998:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879541.817002:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879541.817007:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879541.817010:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879541.817014:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879541.817017:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879542.521117:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879542.521129:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879542.521181:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879542.521182:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879542.521226:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879542.521229:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879542.521233:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879542.841030:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879542.841042:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879542.841047:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879542.841057:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879542.841064:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879542.841071:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879542.841076:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879542.841082:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879542.841088:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879543.545105:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879543.545113:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879543.545120:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879543.545121:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879543.545190:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879543.545193:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879543.545196:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879543.801170:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879543.801183:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879543.801187:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879543.801192:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879543.801198:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879543.801208:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10482 00000100:00080000:2.0:1788879543.801222:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879543.801227:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879543.801230:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879543.801240:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879543.801243:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879543.801247:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879543.801251:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879543.801253:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879543.801255:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879543.801261:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879543.801264:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879543.801268:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879543.801273:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879543.801278:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879543.801281:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879543.801284:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879543.801299:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879543.801302:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879543.801306:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879543.801310:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879543.801317:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879543.801331:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879543.801343:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290173824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879543.801358:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879543.801364:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879543.801392:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879543.801397:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879543.801401:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879543.801403:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879543.801405:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879543.801409:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879543.801412:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10482 00000100:00080000:2.0:1788879543.801417:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879543.801421:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879543.801424:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879543.801429:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879543.801430:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879543.801434:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879543.801438:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879543.801440:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879543.801442:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879543.801447:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879543.801451:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879543.801453:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879543.801463:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879543.801464:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879543.801477:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.801481:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879543.801484:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879543.801487:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:2.0:1788879543.801493:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879543.801494:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879543.801504:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788879543.801505:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879543.801525:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879543.801530:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879543.801534:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879543.801538:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879543.801539:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879543.801563:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879543.801564:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879543.801574:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879543.801577:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879543.801587:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290173824:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879543.801592:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.801608:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:2.0:1788879543.801614:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290173952/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879543.801619:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879543.801628:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879543.801645:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879543.801646:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879543.801659:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879543.801665:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879543.801668:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879543.801675:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879543.801679:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879543.801684:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879543.801686:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879543.801690:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879543.801695:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10482 00000100:00000001:3.0:1788879543.801698:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788879543.801700:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879543.801703:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:2.0:1788879543.801706:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879543.801716:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879543.801717:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1788879543.801730:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879543.801733:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879543.801735:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879543.801739:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879543.801742:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879543.801744:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879543.801751:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879543.801756:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879543.801758:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879543.801762:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879543.801765:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879543.801767:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879543.801770:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879543.801774:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879543.801777:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879543.801783:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879543.801785:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879543.801787:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879543.801796:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000800:00000100:3.0:1788879543.801802:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:2.0:1788879543.801805:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879543.801816:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.801827:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00100000:2.0:1788879543.801829:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290174080/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879543.801840:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879543.801849:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879543.801867:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879543.801867:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879543.801887:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.801889:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879543.801894:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.801899:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305155625 00000100:00000001:3.0:1788879543.801971:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1788879543.801974:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879543.801982:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879543.801985:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879543.801988:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879543.801993:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305155626 00000100:00000001:2.0:1788879543.801996:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879543.802005:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1788879543.802007:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10482 00000100:00000001:3.0:1788879543.802022:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788879543.802023:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879543.802032:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879543.802034:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879543.802038:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879543.802042:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879543.802044:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879543.802046:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879543.802056:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:2.0:1788879543.802058:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879543.802063:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879543.802066:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:3.0:1788879543.802071:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879543/real 1788879543] req@ffff94e5045e5880 x1875765290173824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879543.802078:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879543.802089:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879543.802104:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879543.802114:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:3.0:1788879543.802118:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1788879543.802122:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879543.802124:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879543.802152:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879543.802162:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879543.802164:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000008:00000001:2.0:1788879543.802176:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802176:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879543.802187:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879543.802188:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802208:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1788879543.802211:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879543.802213:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879543.802218:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879543.802224:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1788879543.802227:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290173824/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879543.802229:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879543.802238:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879543.802253:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879543.802270:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290174208/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879543.802278:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879543.802290:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879543.802296:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802297:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879543.802301:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802321:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.802326:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879543.802331:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.802337:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879543.802341:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879543.802347:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879543.802357:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879543.802361:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879543.802368:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290173824:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879543.802375:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.802378:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879543.802381:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879543.802385:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879543.802391:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879543.802395:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879543.802399:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879543.802406:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879543.802408:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879543.802413:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879543.802415:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.802417:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.802428:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879543.802430:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802434:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879543.802436:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879543.802439:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879543.802441:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879543.802443:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879543.802446:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879543.802449:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879543.802452:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879543.802456:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290173952:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879543.802461:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879543.802465:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879543.802467:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879543.802469:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879543.802474:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802486:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879543.802488:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 00000100:00000001:3.0:1788879543.802495:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879543.802517:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879543.802523:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879543.802526:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879543.802528:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802530:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879543.802533:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802540:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305155084 00000100:00000001:3.0:1788879543.802544:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305155625 00000100:00000001:3.0:1788879543.802547:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305155084 00000100:00000001:3.0:1788879543.802550:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305155626 00000100:00000001:3.0:1788879543.802553:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879543.802555:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802557:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879543.802563:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879543/real 1788879543] req@ffff94e5045e5180 x1875765290173952/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879543.802574:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879543.802576:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879543.802578:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879543.802580:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879543.802582:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.802585:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879543.802586:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879543.802591:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290173952/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879543.802600:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.802602:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879543.802605:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.802607:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879543.802609:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.802612:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879543.802631:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879543.802634:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879543.802638:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879543.802641:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879543.802646:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290173952:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879543.802650:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879543.802652:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879543.802654:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879543.802657:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879543.802660:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879543.802662:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879543.802666:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290174080:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879543.802670:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879543.802673:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879543.802675:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879543.802677:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879543.802680:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802687:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879543.802688:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:3.0:1788879543.802692:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879543.802704:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879543.802709:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879543.802711:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879543.802713:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802715:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879543.802718:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802722:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305155082 00000100:00000001:3.0:1788879543.802725:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305155625 00000100:00000001:3.0:1788879543.802728:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305155082 00000100:00000001:3.0:1788879543.802731:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305155626 00000100:00000001:3.0:1788879543.802736:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879543.802738:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802740:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879543.802745:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879543/real 1788879543] req@ffff94e5045e4a80 x1875765290174080/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879543.802754:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879543.802755:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879543.802758:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879543.802760:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879543.802761:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.802763:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879543.802765:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879543.802769:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290174080/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879543.802778:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.802781:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879543.802783:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.802785:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879543.802787:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.802789:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879543.802803:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879543.802806:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879543.802810:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879543.802812:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879543.802816:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290174080:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879543.802820:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879543.802822:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879543.802824:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879543.802827:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879543.802829:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879543.802832:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879543.802836:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290174208:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879543.802840:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879543.802842:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879543.802844:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879543.802847:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879543.802850:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802855:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879543.802857:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:3.0:1788879543.802861:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879543.802872:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879543.802876:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879543.802878:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879543.802880:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802882:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879543.802885:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802888:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305155082 00000100:00000001:3.0:1788879543.802891:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305155625 00000100:00000001:3.0:1788879543.802894:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305155082 00000100:00000001:3.0:1788879543.802897:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305155626 00000100:00000001:3.0:1788879543.802899:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879543.802934:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.802937:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879543.802942:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879543/real 1788879543] req@ffff94e5045e7100 x1875765290174208/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879543.802953:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879543.802954:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879543.802958:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879543.802960:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879543.802962:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.802964:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879543.802966:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879543.802971:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290174208/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879543.802989:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.802992:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879543.802995:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.802997:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879543.802998:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.803000:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879543.803015:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879543.803018:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879543.803021:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879543.803024:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879543.803028:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290174208:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879543.803032:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.803035:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879543.803037:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879543.803039:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879543.803041:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879543.803043:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879543.803045:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879543.803048:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879543.803050:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879543.803052:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879543.803054:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.803056:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879543.803057:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879543.803059:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879543.803062:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879543.803063:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879543.803065:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879543.803066:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879543.803068:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879543.803070:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879543.803071:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.803073:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879543.803075:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879543.803077:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879543.803079:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879543.803081:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879543.803082:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879543.803084:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879543.803086:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879543.803088:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879543.803090:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.803092:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879543.803095:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879543.803097:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879543.803099:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879543.803101:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879543.865020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879543.865028:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879543.865031:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879543.865037:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879543.865042:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879543.865047:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879543.865051:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879543.865055:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879543.865059:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879544.569003:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879544.569011:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879544.569016:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879544.569020:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879544.569023:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879544.569027:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879544.569030:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879544.569032:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879544.569034:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879544.569035:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879544.569101:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879544.569107:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879544.569113:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879544.888986:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879544.888993:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879544.888996:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879544.889000:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879544.889004:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879544.889008:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879544.889011:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879544.889014:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879544.889016:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879545.592983:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879545.592989:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879545.592993:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879545.592994:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879545.593010:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879545.593012:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879545.593015:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879545.913017:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879545.913026:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879545.913029:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879545.913036:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879545.913041:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879545.913046:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879545.913050:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879545.913054:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879545.913057:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879546.617022:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879546.617031:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879546.617039:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879546.617043:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879546.617071:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879546.617075:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879546.617078:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879546.936998:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879546.937004:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879546.937006:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879546.937010:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879546.937013:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879546.937016:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879546.937019:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879546.937021:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879546.937023:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879547.640998:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879547.641005:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879547.641007:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879547.641010:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879547.641014:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879547.641016:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879547.641017:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879547.641019:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879547.641022:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879547.641023:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879547.641040:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879547.641042:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879547.641044:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879547.960971:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879547.960977:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879547.960978:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879547.960982:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879547.960985:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879547.960988:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879547.960990:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879547.960993:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879547.960995:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879548.665051:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879548.665062:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879548.665072:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879548.665075:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879548.665132:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879548.665136:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879548.665140:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879548.921023:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879548.921032:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879548.921034:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879548.921036:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879548.921040:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879548.921046:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10487 00000100:00080000:0.0:1788879548.921055:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879548.921058:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879548.921060:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879548.921067:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879548.921069:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879548.921071:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879548.921073:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879548.921075:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879548.921076:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879548.921079:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879548.921081:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879548.921083:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879548.921086:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879548.921088:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921090:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879548.921102:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879548.921113:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879548.921114:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921117:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879548.921119:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879548.921122:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921132:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879548.921140:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290174336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879548.921148:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879548.921151:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921163:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879548.921165:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879548.921168:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879548.921169:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879548.921171:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879548.921173:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879548.921174:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10487 00000100:00080000:0.0:1788879548.921177:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879548.921179:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879548.921180:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879548.921182:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879548.921183:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879548.921184:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879548.921186:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879548.921188:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879548.921189:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879548.921192:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879548.921194:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879548.921195:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879548.921199:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879548.921201:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921202:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879548.921205:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879548.921207:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:0.0:1788879548.921208:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921212:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921214:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788879548.921216:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879548.921219:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921219:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879548.921224:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879548.921224:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788879548.921225:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879548.921229:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879548.921230:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921238:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879548.921238:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879548.921243:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788879548.921245:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897aa00 x1875765290174464/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879548.921248:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921257:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:3.0:1788879548.921258:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290174336:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879548.921260:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921266:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921268:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1788879548.921269:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879548.921273:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:3.0:1788879548.921273:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788879548.921276:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:3.0:1788879548.921278:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879548.921279:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879548.921285:0:80852:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:3.0:1788879548.921285:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1788879548.921289:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10487 00000100:00080000:0.0:1788879548.921291:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1788879548.921292:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921293:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879548.921295:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879548.921298:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879548.921299:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879548.921301:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879548.921302:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879548.921304:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879548.921305:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879548.921307:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879548.921308:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879548.921309:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879548.921312:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879548.921312:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1788879548.921316:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921316:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:0.0:1788879548.921320:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879548.921323:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879548.921324:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:0.0:1788879548.921325:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921328:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879548.921332:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879548.921333:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921334:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879548.921335:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879548.921336:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921340:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879548.921344:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978700 x1875765290174592/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879548.921349:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879548.921351:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921353:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879548.921355:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879548.921357:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879548.921359:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879548.921360:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879548.921362:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879548.921364:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10487 00000100:00080000:0.0:1788879548.921366:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879548.921368:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879548.921368:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879548.921369:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879548.921371:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879548.921374:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879548.921375:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879548.921375:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305160745 00000020:00000001:0.0:1788879548.921380:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879548.921382:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879548.921401:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879548.921407:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1788879548.921408:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879548.921419:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879548.921422:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879548.921426:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305160745 02000000:00000001:0.0:1788879548.921427:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879548.921438:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921440:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879548.921443:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879548.921446:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000008:00000001:0.0:1788879548.921447:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921457:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921461:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879548.921465:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879548.921472:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921472:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921477:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879548.921483:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879548.921486:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879548.921487:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921500:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879548.921503:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879548.921512:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290174720/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879548.921514:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879548.921537:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788879548.921548:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879548.921555:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879548.921562:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879548.921564:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879548.921571:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879548.921572:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921579:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879548.921581:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879548.921591:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879548.921598:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879548.921604:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879548.921615:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290174464:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879548.921624:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879548.921643:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879548.921644:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879548.921646:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879548.921649:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921655:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879548.921656:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:3.0:1788879548.921660:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879548.921671:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921675:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305160203 00000100:00000001:3.0:1788879548.921677:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305160745 00000100:00000001:3.0:1788879548.921679:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305160203 00000100:00000001:3.0:1788879548.921681:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305160745 00000100:00000001:3.0:1788879548.921683:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879548.921684:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921686:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879548.921687:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879548.921688:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879548.921689:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879548.921691:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879548.921693:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879548.921696:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290174592:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879548.921699:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879548.921700:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879548.921701:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879548.921703:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879548.921704:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921707:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879548.921708:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00000001:3.0:1788879548.921711:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879548.921718:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921721:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305160201 00000100:00000001:3.0:1788879548.921724:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305160745 00000100:00000001:3.0:1788879548.921726:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305160201 00000100:00000001:3.0:1788879548.921728:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305160745 00000100:00000001:3.0:1788879548.921730:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879548.921733:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921735:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921737:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879548.921741:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879548.921742:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788879548.921745:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879548.921746:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879548.921748:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879548.921749:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879548.921750:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879548.921753:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879548.921754:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879548.921757:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879548.921761:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290174720:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879548.921765:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879548.921783:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879548.921785:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879548.921789:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879548.921795:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921806:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879548.921808:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:3.0:1788879548.921813:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879548.921833:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921840:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305160201 00000100:00000001:3.0:1788879548.921844:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305160745 00000100:00000001:3.0:1788879548.921848:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305160201 00000100:00000001:3.0:1788879548.921852:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305160745 00000100:00000001:3.0:1788879548.921856:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879548.921859:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921864:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921867:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879548.921873:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879548.921876:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788879548.921879:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879548.921881:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879548.921884:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879548.921888:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879548.984993:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879548.985010:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879548.985012:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879548.985016:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879548.985019:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879548.985022:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879548.985025:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879548.985026:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879548.985029:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879549.689038:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879549.689049:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879549.689061:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879549.689063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879549.689118:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879549.689122:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879549.689126:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879550.009007:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879550.009026:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879550.009029:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879550.009035:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879550.009039:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879550.009044:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879550.009048:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879550.009052:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879550.009057:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879550.713034:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879550.713045:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879550.713054:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879550.713057:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879550.713119:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879550.713125:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879550.713131:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879551.033044:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879551.033053:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879551.033056:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879551.033061:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879551.033066:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879551.033070:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879551.033073:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879551.033114:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879551.033121:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879551.737099:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879551.737105:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879551.737110:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879551.737111:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879551.737130:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879551.737133:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879551.737135:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788879551.993416:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1788879551.993499:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788879551.993528:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879551.993578:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879551.993610:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879551.993615:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879551.993620:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879551.993634:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879551.993641:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879551.993645:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879551.993649:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879551.993652:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879551.993662:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879551.993673:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879551.993678:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879551.993682:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879551.993685:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879551.993687:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000100:0.0:1788879551.993699:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788879551.993699:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879551.993709:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879551.993714:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879551.993724:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879551.993730:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879551.993734:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879548/real 1788879551] req@ffff94e50897aa00 x1875765290174464/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879551.993735:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000400:00000001:3.0:1788879551.993765:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879551.993770:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879551.993780:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879551.993786:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879551.993789:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.993795:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879551.993799:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879551.993808:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290174464/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879551.993833:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.993858:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879551.993865:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.993869:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879551.993871:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.993877:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879551.993939:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879551.993948:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879551.993958:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879551.993965:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879551.993975:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290174464:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879551.993985:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879551.993994:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879548/real 1788879551] req@ffff94e508978700 x1875765290174592/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879551.994023:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879551.994025:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879551.994032:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879551.994036:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879551.994038:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.994042:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879551.994044:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879551.994053:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290174592/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879551.994065:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.994091:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879551.994097:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.994101:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879551.994103:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.994107:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879551.994130:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879551.994135:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879551.994144:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879551.994149:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879551.994157:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290174592:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879551.994175:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879551.994185:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879548/real 1788879551] req@ffff94e508978a80 x1875765290174336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879551.994200:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879551.994203:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879551.994209:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879551.994212:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879551.994215:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.994219:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879551.994221:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879551.994229:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290174336/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879551.994243:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.994247:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879551.994251:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.994254:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879551.994257:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.994260:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879551.994263:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879551.994278:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879551.994286:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879551.994291:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879551.994299:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290174336:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879551.994312:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879551.994316:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879551.994319:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879551.994325:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879551.994334:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879551.994338:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879551.994341:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879551.994347:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879551.994351:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879551.994357:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879551.994360:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.994363:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879551.994365:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879551.994367:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879551.994387:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879551.994390:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879551.994392:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879551.994396:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879551.994398:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879551.994401:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879551.994404:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.994407:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879551.994409:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879551.994411:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879551.994415:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879551.994416:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879551.994418:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879551.994421:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879551.994423:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879551.994426:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879551.994428:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.994432:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879551.994435:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879551.994453:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879551.994456:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879551.994463:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879548/real 1788879551] req@ffff94e50897ad80 x1875765290174720/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879551.994478:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879551.994481:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879551.994486:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879551.994489:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879551.994491:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.994494:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879551.994813:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879551.994825:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290174720/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879551.994836:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.994839:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879551.994861:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.994864:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879551.994872:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.994876:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879551.994932:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879551.994940:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879551.994949:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879551.994955:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879551.994961:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290174720:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879551.994969:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.994972:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879551.994975:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879551.994978:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879551.994982:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879551.994985:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879551.994987:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879551.994991:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879551.994993:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879551.994998:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879551.995000:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.995002:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879551.995011:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879551.995014:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879551.995017:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879551.995019:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879552.057023:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879552.057033:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879552.057036:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879552.057042:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879552.057047:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879552.057052:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879552.057056:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879552.057060:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879552.057063:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879552.761042:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879552.761049:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879552.761055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879552.761057:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879552.761090:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879552.761093:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879552.761095:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879553.080988:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879553.080996:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879553.080998:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879553.081002:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879553.081005:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879553.081009:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879553.081012:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879553.081015:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879553.081018:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879553.785077:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879553.785086:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879553.785094:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879553.785096:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879553.785136:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879553.785139:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879553.785142:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879554.041163:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879554.041181:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879554.041187:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879554.041196:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879554.041205:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879554.041219:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10492 00000100:00080000:0.0:1788879554.041240:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879554.041247:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879554.041251:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879554.041266:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879554.041272:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879554.041278:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879554.041284:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879554.041295:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879554.041298:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879554.041312:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879554.041319:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879554.041326:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879554.041334:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879554.041341:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879554.041346:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879554.041352:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879554.041371:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879554.041377:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879554.041384:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879554.041390:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879554.041399:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879554.041420:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879554.041439:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290174848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879554.041461:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879554.041469:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879554.041507:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879554.041515:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879554.041522:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879554.041525:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879554.041530:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879554.041537:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879554.041549:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10492 00000100:00080000:0.0:1788879554.041560:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879554.041566:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879554.041569:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879554.041574:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788879554.041579:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879554.041587:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879554.041588:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788879554.041594:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879554.041598:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1788879554.041604:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879554.041608:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879554.041614:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879554.041617:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879554.041623:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879554.041635:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1788879554.041639:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:2.0:1788879554.041645:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879554.041650:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879554.041656:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879554.041659:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879554.041674:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1788879554.041679:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290174848:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788879554.041684:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.041695:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879554.041705:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879554.041715:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879554.041718:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:2.0:1788879554.041729:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:0.0:1788879554.041733:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788879554.041745:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788879554.041746:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879554.041763:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879554.041778:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879554.041781:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879554.041786:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879554.041790:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879554.041795:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1788879554.041796:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.041805:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:0.0:1788879554.041817:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.041821:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1788879554.041843:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290174976/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879554.041862:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879554.041868:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879554.041877:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879554.041883:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879554.041891:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879554.041895:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879554.041898:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879554.041968:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879554.041975:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10492 00000100:00080000:0.0:1788879554.041987:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879554.041992:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879554.041997:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879554.042012:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000100:2.0:1788879554.042013:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:0.0:1788879554.042021:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879554.042030:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879554.042038:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879554.042044:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879554.042047:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879554.042048:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879554.042090:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1788879554.042093:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879554.042098:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879554.042105:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.042108:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:0.0:1788879554.042118:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879554.042126:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879554.042128:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879554.042140:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.042143:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305165866 00000100:00000001:0.0:1788879554.042146:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879554.042156:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879554.042164:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879554.042168:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.042170:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1788879554.042184:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879554.042189:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305165866 02000000:00000001:0.0:1788879554.042192:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879554.042201:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879554.042207:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879554.042207:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1788879554.042216:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.042221:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879554.042231:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879554.042235:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.042237:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879554.042270:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879554/real 1788879554] req@ffff94e508978e00 x1875765290174848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1788879554.042272:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290175104/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1788879554.042319:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879554.042325:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879554.042337:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879554.042343:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879554.042346:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879554.042352:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.042357:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879554.042359:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879554.042372:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290174848/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879554.042384:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.042404:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879554.042405:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.042415:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1788879554.042418:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879554.042431:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879554.042437:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879554.042442:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879554.042446:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879554.042450:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879554.042456:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879554.042459:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879554.042467:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879554.042471:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879554.042482:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1788879554.042486:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10492 00000100:00080000:2.0:1788879554.042499:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879554.042519:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:0.0:1788879554.042522:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879554.042535:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1788879554.042536:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290174848:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879554.042543:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879554.042552:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788879554.042557:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879554.042563:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1788879554.042566:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879554.042575:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879554.042579:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1788879554.042583:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:2.0:1788879554.042590:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1788879554.042594:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879554.042602:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879554.042603:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879554.042614:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00080000:0.0:1788879554.042618:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000020:00000001:2.0:1788879554.042622:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:0.0:1788879554.042628:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879554.042635:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1788879554.042637:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879554.042643:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:0.0:1788879554.042649:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879554.042656:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 02000000:00000001:0.0:1788879554.042658:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.042663:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879554.042668:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879554.042674:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1788879554.042678:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879554.042690:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.042693:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879554.042696:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.042701:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.042718:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788879554.042719:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879554.042726:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788879554.042728:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.042734:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879554.042738:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879554.042742:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788879554.042747:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879554.042750:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879554.042760:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879554.042764:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879554.042769:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879554.042770:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879554.042792:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290174976:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788879554.042795:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.042811:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879554.042817:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:0.0:1788879554.042824:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290175232/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788879554.042825:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879554.042836:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879554.042846:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879554.042862:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879554.042868:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879554.042870:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.042877:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:0.0:1788879554.042885:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.042891:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879554.042982:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879554.043006:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879554.043013:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879554.043017:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.043021:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879554.043026:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.043034:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305165325 00000100:00000001:2.0:1788879554.043039:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305165866 00000100:00000001:2.0:1788879554.043044:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305165325 00000100:00000001:2.0:1788879554.043048:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305165866 00000100:00000001:2.0:1788879554.043052:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879554.043079:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.043083:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879554.043093:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879554/real 1788879554] req@ffff94e50897a300 x1875765290174976/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879554.043108:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879554.043111:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879554.043116:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879554.043119:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879554.043121:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043124:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879554.043126:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879554.043133:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290174976/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879554.043144:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043148:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879554.043152:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043155:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879554.043157:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043160:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879554.043181:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879554.043186:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879554.043191:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879554.043195:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879554.043200:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290174976:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879554.043207:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879554.043209:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879554.043212:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879554.043216:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879554.043219:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879554.043222:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879554.043227:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290175104:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879554.043233:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879554.043236:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879554.043240:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879554.043243:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879554.043248:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.043257:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879554.043259:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:2.0:1788879554.043265:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879554.043287:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879554.043295:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879554.043298:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879554.043301:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.043304:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879554.043308:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.043314:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305165323 00000100:00000001:2.0:1788879554.043319:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305165866 00000100:00000001:2.0:1788879554.043323:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305165323 00000100:00000001:2.0:1788879554.043328:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305165866 00000100:00000001:2.0:1788879554.043333:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879554.043337:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.043341:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879554.043350:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879554/real 1788879554] req@ffff94e508979c00 x1875765290175104/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879554.043365:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879554.043367:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879554.043372:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879554.043375:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879554.043377:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043379:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879554.043381:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879554.043385:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290175104/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879554.043394:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043399:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879554.043403:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043404:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879554.043406:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043409:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879554.043425:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879554.043428:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879554.043433:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879554.043435:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879554.043439:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290175104:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879554.043443:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043445:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879554.043446:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879554.043448:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879554.043461:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879554.043463:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879554.043465:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879554.043470:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879554.043473:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879554.043478:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879554.043480:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043484:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879554.043487:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879554.043489:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879554.043494:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879554.043496:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879554.043497:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879554.043501:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879554.043502:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879554.043506:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879554.043509:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043513:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043521:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879554.043524:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.043528:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879554.043531:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879554.043533:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879554.043536:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879554.043540:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879554.043544:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879554.043547:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879554.043550:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879554.043557:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290175232:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879554.043563:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879554.043566:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879554.043569:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879554.043572:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879554.043578:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.043589:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879554.043592:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:2.0:1788879554.043598:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879554.043632:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879554.043641:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879554.043644:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879554.043646:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.043648:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879554.043651:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.043656:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305165323 00000100:00000001:2.0:1788879554.043659:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305165866 00000100:00000001:2.0:1788879554.043662:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305165323 00000100:00000001:2.0:1788879554.043665:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305165866 00000100:00000001:2.0:1788879554.043669:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879554.043673:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.043676:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879554.043684:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879554/real 1788879554] req@ffff94e50897bb80 x1875765290175232/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879554.043698:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879554.043700:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879554.043705:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879554.043709:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879554.043711:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043716:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879554.043719:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879554.043726:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290175232/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879554.043738:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043742:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879554.043746:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043749:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879554.043752:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043755:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879554.043771:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879554.043775:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879554.043798:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879554.043801:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879554.043805:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290175232:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879554.043810:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043813:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879554.043814:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879554.043816:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879554.043819:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879554.043821:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879554.043823:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879554.043826:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879554.043827:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879554.043831:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879554.043834:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043836:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879554.043840:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879554.043844:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879554.043848:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879554.043851:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879554.105003:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879554.105014:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879554.105017:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879554.105025:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879554.105030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879554.105035:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879554.105039:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879554.105042:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879554.105046:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879554.809083:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879554.809091:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879554.809097:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879554.809102:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879554.809107:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879554.809112:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879554.809116:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879554.809119:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879554.809123:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879554.809127:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879554.809156:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879554.809164:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879554.809168:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879555.129081:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879555.129091:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879555.129093:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879555.129101:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879555.129106:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879555.129111:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879555.129115:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879555.129119:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879555.129122:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879555.833036:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879555.833074:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879555.833082:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879555.833084:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879555.833110:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879555.833113:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879555.833118:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879556.153083:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879556.153095:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879556.153100:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879556.153109:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879556.153116:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879556.153122:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879556.153128:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879556.153133:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879556.153138:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879556.857114:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879556.857125:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879556.857135:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879556.857138:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879556.857166:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879556.857169:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879556.857173:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879557.177140:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879557.177149:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879557.177153:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879557.177160:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879557.177165:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879557.177170:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879557.177173:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879557.177177:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879557.177180:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879557.881131:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879557.881138:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879557.881140:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879557.881143:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879557.881148:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879557.881150:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879557.881152:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879557.881153:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879557.881157:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879557.881158:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879557.881176:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879557.881178:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879557.881180:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879558.201094:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879558.201102:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879558.201104:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879558.201110:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879558.201113:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879558.201118:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879558.201121:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879558.201124:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879558.201127:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879558.905062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879558.905068:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879558.905073:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879558.905075:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879558.905097:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879558.905102:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879558.905105:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879559.161069:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879559.161081:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879559.161085:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879559.161090:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879559.161095:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879559.161104:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10498 00000100:00080000:2.0:1788879559.161117:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879559.161120:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879559.161123:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879559.161134:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879559.161137:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879559.161141:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879559.161144:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879559.161146:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879559.161147:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879559.161152:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879559.161154:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879559.161157:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879559.161161:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879559.161165:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879559.161167:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879559.161169:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879559.161184:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879559.161186:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879559.161190:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879559.161193:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879559.161199:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879559.161214:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879559.161224:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290175360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879559.161236:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879559.161241:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879559.161264:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879559.161267:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879559.161273:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879559.161276:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879559.161278:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879559.161281:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879559.161284:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10498 00000100:00080000:2.0:1788879559.161288:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879559.161290:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879559.161291:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879559.161294:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879559.161295:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879559.161297:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879559.161300:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879559.161302:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879559.161320:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879559.161323:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879559.161325:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879559.161328:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879559.161330:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879559.161332:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879559.161341:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:3.0:1788879559.161344:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879559.161346:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879559.161350:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879559.161351:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879559.161358:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879559.161362:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879559.161370:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879559.161371:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290175360:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879559.161376:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879559.161386:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:2.0:1788879559.161387:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788879559.161391:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:2.0:1788879559.161394:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879559.161396:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879559.161399:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879559.161402:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879559.161407:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879559.161409:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879559.161413:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879559.161415:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879559.161418:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879559.161418:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879559.161440:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.161450:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00100000:2.0:1788879559.161451:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290175488/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879559.161454:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:3.0:1788879559.161467:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879559.161487:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879559.161491:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879559.161497:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879559.161503:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879559.161507:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879559.161510:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879559.161512:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879559.161523:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000800:00000100:3.0:1788879559.161523:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788879559.161534:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10498 00000100:00000001:3.0:1788879559.161546:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788879559.161547:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879559.161555:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879559.161557:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879559.161558:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879559.161563:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879559.161566:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879559.161568:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:2.0:1788879559.161570:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879559.161574:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879559.161575:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879559.161586:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879559.161608:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305170985 00000100:00000001:2.0:1788879559.161611:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879559.161614:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879559.161619:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879559.161620:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788879559.161630:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879559.161634:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879559.161637:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879559.161638:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305170985 02000000:00000001:2.0:1788879559.161640:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879559.161646:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879559.161648:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000008:00100000:2.0:1788879559.161650:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879559.161653:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1788879559.161654:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879559.161664:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.161665:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.161679:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879559.161686:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879559/real 1788879559] req@ffff94e5045e5c00 x1875765290175360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879559.161710:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1788879559.161716:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000400:00000001:3.0:1788879559.161717:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879559.161723:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879559.161725:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:2.0:1788879559.161734:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.161734:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.161740:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879559.161744:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879559.161752:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1788879559.161753:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290175360/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879559.161761:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879559.161770:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879559.161772:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.161776:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879559.161780:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.161781:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879559.161782:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.161784:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879559.161786:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879559.161787:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879559.161790:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879559.161798:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879559.161801:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879559.161813:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4380 x1875765290175616/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879559.161814:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290175360:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879559.161837:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.161846:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879559.161850:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788879559.161854:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788879559.161857:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788879559.161863:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879559.161866:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1788879559.161872:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879559.161872:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879559.161877:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879559.161895:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:2.0:1788879559.161896:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1788879559.161899:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:2.0:1788879559.161905:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879559.161908:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879559.161912:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879559.161928:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879559.161930:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.161934:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.161942:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879559.161943:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.161945:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879559.161947:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879559.161948:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879559.161952:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879559.161953:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879559.161954:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879559.161959:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1788879559.161961:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10498 02000000:00000001:3.0:1788879559.161963:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788879559.161970:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1788879559.161971:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879559.161982:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879559.161987:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:3.0:1788879559.161988:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290175488:192.168.203.129@tcp:38:kworker.0 00000020:00000001:2.0:1788879559.162003:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879559.162003:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1788879559.162012:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:3.0:1788879559.162014:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879559.162033:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879559.162036:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1788879559.162039:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:3.0:1788879559.162039:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879559.162052:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879559.162056:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879559.162057:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879559.162060:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879559.162061:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:3.0:1788879559.162076:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788879559.162077:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879559.162085:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879559.162088:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879559.162092:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879559.162095:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879559.162098:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879559.162101:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879559.162104:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1788879559.162111:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879559.162112:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879559.162126:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879559.162128:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:2.0:1788879559.162138:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.162142:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879559.162143:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879559.162148:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879559.162149:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879559.162156:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.162156:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879559.162162:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879559.162168:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.162168:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305170444 00000100:00000001:3.0:1788879559.162175:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305170985 00000100:00100000:2.0:1788879559.162180:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290175744/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879559.162181:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305170444 00000100:00000001:3.0:1788879559.162191:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305170985 00000100:00000001:3.0:1788879559.162201:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879559.162214:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879559.162218:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879559.162219:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879559.162223:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879559.162231:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879559.162231:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879559/real 1788879559] req@ffff94e5045e5500 x1875765290175488/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879559.162246:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879559.162247:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879559.162250:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879559.162252:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879559.162254:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162256:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879559.162258:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879559.162262:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290175488/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879559.162267:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162269:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879559.162270:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162271:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879559.162272:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162273:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879559.162287:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879559.162289:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879559.162303:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879559.162305:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879559.162307:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290175488:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879559.162310:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879559.162311:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879559.162312:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879559.162314:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879559.162316:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879559.162317:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879559.162320:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290175616:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879559.162322:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879559.162324:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879559.162325:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879559.162327:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879559.162329:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.162333:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879559.162334:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074549120 : -117763635002496 : ffff94e5045e4380) 00000100:00000001:3.0:1788879559.162337:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879559.162345:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879559.162348:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879559.162350:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879559.162351:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.162352:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879559.162353:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.162356:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305170442 00000100:00000001:3.0:1788879559.162358:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305170985 00000100:00000001:3.0:1788879559.162359:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305170442 00000100:00000001:3.0:1788879559.162361:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305170985 00000100:00000001:3.0:1788879559.162362:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879559.162364:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.162365:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879559.162367:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879559/real 1788879559] req@ffff94e5045e4380 x1875765290175616/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879559.162372:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879559.162373:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879559.162374:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879559.162375:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879559.162376:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162378:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879559.162379:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879559.162381:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4380 x1875765290175616/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879559.162385:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162387:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879559.162388:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162389:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879559.162390:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162391:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879559.162404:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879559.162406:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879559.162408:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879559.162410:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879559.162412:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290175616:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879559.162415:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162417:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879559.162417:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879559.162419:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879559.162420:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879559.162421:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879559.162422:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879559.162423:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879559.162425:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879559.162426:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879559.162427:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162428:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879559.162429:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879559.162430:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879559.162431:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879559.162432:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879559.162433:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879559.162434:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879559.162434:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879559.162435:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879559.162436:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162437:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162439:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879559.162440:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.162442:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879559.162443:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879559.162444:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879559.162445:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879559.162446:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879559.162448:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879559.162449:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879559.162450:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879559.162452:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290175744:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879559.162455:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879559.162457:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879559.162458:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879559.162460:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879559.162461:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.162464:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879559.162465:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000100:00000001:3.0:1788879559.162468:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879559.162474:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879559.162477:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879559.162479:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879559.162480:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.162481:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879559.162483:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.162485:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305170442 00000100:00000001:3.0:1788879559.162487:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305170985 00000100:00000001:3.0:1788879559.162489:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305170442 00000100:00000001:3.0:1788879559.162491:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305170985 00000100:00000001:3.0:1788879559.162492:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879559.162493:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.162494:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879559.162497:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879559/real 1788879559] req@ffff94e5045e7800 x1875765290175744/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879559.162501:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879559.162502:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879559.162504:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879559.162505:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879559.162506:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162507:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879559.162508:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879559.162510:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290175744/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879559.162515:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162516:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879559.162518:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162519:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879559.162520:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162521:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879559.162535:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879559.162537:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879559.162539:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879559.162541:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879559.162543:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290175744:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879559.162545:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162546:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879559.162547:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879559.162548:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879559.162549:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879559.162550:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879559.162550:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879559.162551:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879559.162552:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879559.162553:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879559.162554:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162555:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879559.162557:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879559.162558:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879559.162559:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879559.162560:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879559.224984:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879559.224988:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879559.224989:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879559.224993:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879559.224995:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879559.224997:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879559.224999:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879559.225001:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879559.225002:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879559.929049:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879559.929068:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879559.929075:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879559.929077:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879559.929100:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879559.929102:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879559.929104:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879560.249077:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879560.249084:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879560.249086:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879560.249092:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879560.249097:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879560.249102:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879560.249105:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879560.249108:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879560.249112:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879560.953079:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879560.953087:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879560.953094:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879560.953096:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879560.953122:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879560.953125:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879560.953129:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879561.273104:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879561.273117:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879561.273122:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879561.273132:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879561.273140:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879561.273147:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879561.273151:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879561.273156:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879561.273161:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879561.977120:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879561.977129:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879561.977136:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879561.977138:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879561.977210:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879561.977213:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879561.977218:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879562.297103:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879562.297114:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879562.297115:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879562.297122:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879562.297126:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879562.297131:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879562.297133:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879562.297137:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879562.297140:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879563.001112:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879563.001122:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879563.001129:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879563.001131:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879563.001179:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879563.001183:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879563.001187:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879563.321007:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879563.321032:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879563.321035:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879563.321040:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879563.321043:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879563.321047:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879563.321049:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879563.321052:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879563.321054:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879564.025104:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879564.025115:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879564.025126:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879564.025129:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879564.025183:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879564.025187:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879564.025191:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879564.281118:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879564.281132:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879564.281136:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879564.281141:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879564.281148:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879564.281157:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10503 00000100:00080000:1.0:1788879564.281173:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879564.281179:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879564.281183:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879564.281197:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879564.281201:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879564.281207:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879564.281213:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879564.281218:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879564.281221:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879564.281230:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879564.281236:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879564.281242:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879564.281249:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879564.281272:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879564.281278:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879564.281282:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879564.281301:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879564.281305:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879564.281310:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879564.281315:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879564.281323:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879564.281345:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879564.281362:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100ad80 x1875765290175872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879564.281383:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879564.281390:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879564.281418:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879564.281424:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879564.281431:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879564.281434:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879564.281438:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879564.281444:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879564.281449:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10503 00000100:00080000:1.0:1788879564.281458:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879564.281463:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879564.281466:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879564.281473:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879564.281477:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879564.281482:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879564.281487:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879564.281491:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879564.281494:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879564.281502:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1788879564.281503:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879564.281513:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879564.281515:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.281523:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1788879564.281525:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879564.281534:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788879564.281540:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879564.281542:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879564.281551:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879564.281557:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788879564.281561:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788879564.281566:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:1.0:1788879564.281569:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879564.281580:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879564.281582:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879564.281584:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879564.281600:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879564.281603:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879564.281605:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290175872:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879564.281607:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879564.281613:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879564.281619:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.281626:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788879564.281632:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879564.281637:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879564.281648:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879564.281649:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008000 x1875765290176000/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879564.281662:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788879564.281671:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879564.281675:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879564.281677:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879564.281681:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879564.281688:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879564.281692:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879564.281694:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879564.281696:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879564.281701:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879564.281704:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10503 00000100:00000001:3.0:1788879564.281710:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1788879564.281711:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879564.281721:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879564.281722:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286519680 : -117763423031936 : ffff94e51100ad80) 00000100:00000001:1.0:1788879564.281724:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879564.281738:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879564.281739:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1788879564.281741:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879564.281748:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879564.281751:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879564.281754:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879564.281756:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879564.281771:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879564.281775:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879564.281777:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879564.281782:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879564.281785:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879564.281789:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879564.281791:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879564.281794:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879564.281797:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879564.281805:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879564.281808:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879564.281813:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000100:3.0:1788879564.281817:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:1.0:1788879564.281818:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879564.281824:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879564.281832:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879564.281839:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008e00 x1875765290176128/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879564.281845:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879564.281856:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879564.281860:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.281860:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879564.281872:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.281872:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879564.281883:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879564.281886:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:1.0:1788879564.281890:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879564.281896:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879564.281899:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879564.281970:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879564.281975:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879564.281976:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10503 00000100:00080000:1.0:1788879564.282013:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879564.282039:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879564.282042:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305176106 00000100:00000001:1.0:1788879564.282044:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879564.282057:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879564.282059:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879564.282062:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:1.0:1788879564.282067:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879564.282068:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788879564.282070:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879564.282079:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879564.282084:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879564.282087:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.282088:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305176106 02000000:00000001:1.0:1788879564.282090:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879564.282097:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879564.282103:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1788879564.282104:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879564.282110:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879564.282115:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879564.282117:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:1.0:1788879564.282119:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879564.282125:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.282131:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879564.282135:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1788879564.282140:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1788879564.282141:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879564.282149:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879564.282151:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879564.282154:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879564.282161:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879564/real 1788879564] req@ffff94e51100ad80 x1875765290175872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879564.282164:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879564.282196:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008700 x1875765290176256/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879564.282236:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:3.0:1788879564.282239:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1788879564.282240:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879564.282247:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879564.282249:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.282263:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879564.282272:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879564.282276:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.282283:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879564.282288:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879564.282301:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100ad80 x1875765290175872/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879564.282319:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.282325:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879564.282331:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.282336:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879564.282340:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.282362:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879564.282366:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879564.282372:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879564.282379:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879564.282384:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879564.282392:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290175872:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879564.282401:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.282405:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879564.282407:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879564.282428:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879564.282435:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879564.282438:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879564.282442:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879564.282449:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879564.282453:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879564.282459:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879564.282461:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.282466:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.282477:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879564.282482:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.282487:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879564.282490:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879564.282494:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879564.282497:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879564.282500:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879564.282505:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879564.282509:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879564.282514:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879564.282521:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290176000:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879564.282529:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879564.282533:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879564.282537:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879564.282541:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879564.282547:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.282559:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879564.282562:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508032 : -117763423043584 : ffff94e511008000) 00000100:00000001:3.0:1788879564.282570:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879564.282596:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879564.282621:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879564.282626:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879564.282629:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.282632:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879564.282637:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.282644:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305175564 00000100:00000001:3.0:1788879564.282649:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305176106 00000100:00000001:3.0:1788879564.282653:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305175564 00000100:00000001:3.0:1788879564.282658:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305176106 00000100:00000001:3.0:1788879564.282662:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879564.282666:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.282685:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879564.282693:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879564/real 1788879564] req@ffff94e511008000 x1875765290176000/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879564.282712:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879564.282715:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879564.282719:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879564.282723:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879564.282725:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.282729:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879564.282732:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879564.282740:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008000 x1875765290176000/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879564.282755:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.282760:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879564.282765:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.282768:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879564.282772:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.282776:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879564.282801:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879564.282808:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879564.282815:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879564.282819:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879564.282843:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290176000:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879564.282851:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879564.282854:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879564.282857:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879564.282861:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879564.282865:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879564.282869:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879564.282874:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290176128:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879564.282881:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879564.282884:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879564.282887:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879564.282890:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879564.282896:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.282906:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879564.282909:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286511616 : -117763423040000 : ffff94e511008e00) 00000100:00000001:3.0:1788879564.282915:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879564.282938:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879564.283006:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879564.283010:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879564.283013:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.283016:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879564.283022:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.283028:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305175563 00000100:00000001:3.0:1788879564.283033:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305176106 00000100:00000001:3.0:1788879564.283037:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305175563 00000100:00000001:3.0:1788879564.283040:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305176106 00000100:00000001:3.0:1788879564.283045:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879564.283048:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.283052:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879564.283059:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879564/real 1788879564] req@ffff94e511008e00 x1875765290176128/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879564.283074:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879564.283077:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879564.283081:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879564.283084:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879564.283086:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.283089:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879564.283092:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879564.283098:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008e00 x1875765290176128/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879564.283111:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.283116:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879564.283121:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.283124:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879564.283127:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.283130:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879564.283153:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879564.283157:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879564.283163:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879564.283167:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879564.283172:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290176128:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879564.283179:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879564.283181:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879564.283184:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879564.283187:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879564.283190:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879564.283194:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879564.283199:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290176256:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879564.283204:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879564.283207:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879564.283210:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879564.283214:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879564.283219:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.283226:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879564.283229:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286509824 : -117763423041792 : ffff94e511008700) 00000100:00000001:3.0:1788879564.283234:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879564.283254:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879564.283261:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879564.283265:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879564.283267:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.283270:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879564.283275:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.283280:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305175563 00000100:00000001:3.0:1788879564.283284:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305176106 00000100:00000001:3.0:1788879564.283288:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305175563 00000100:00000001:3.0:1788879564.283292:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305176106 00000100:00000001:3.0:1788879564.283296:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879564.283299:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.283318:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879564.283325:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879564/real 1788879564] req@ffff94e511008700 x1875765290176256/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879564.283346:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879564.283348:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879564.283353:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879564.283356:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879564.283358:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.283361:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879564.283363:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879564.283370:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008700 x1875765290176256/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879564.283394:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.283416:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879564.283420:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.283423:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879564.283426:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.283430:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879564.283454:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879564.283459:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879564.283465:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879564.283469:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879564.283475:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290176256:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879564.283483:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.283488:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879564.283490:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879564.283493:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879564.283498:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879564.283501:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879564.283504:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879564.283509:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879564.283511:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879564.283515:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879564.283517:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.283521:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879564.283523:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879564.283526:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879564.283529:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879564.283531:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879564.283533:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879564.283536:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879564.283539:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879564.283542:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879564.283545:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.283548:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879564.283550:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879564.283553:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879564.283556:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879564.283559:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879564.283561:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879564.283565:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879564.283567:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879564.283569:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879564.283572:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.283576:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879564.283583:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879564.283587:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879564.283591:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879564.283595:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879564.345141:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879564.345151:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879564.345165:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879564.345171:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879564.345176:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879564.345181:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879564.345184:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879564.345188:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879564.345191:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879565.049065:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879565.049073:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879565.049080:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879565.049084:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879565.049087:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879565.049091:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879565.049095:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879565.049097:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879565.049099:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879565.049102:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879565.049127:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879565.049130:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879565.049134:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879565.369072:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879565.369086:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879565.369091:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879565.369101:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879565.369110:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879565.369117:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879565.369123:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879565.369129:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879565.369134:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879566.073055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879566.073063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879566.073069:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879566.073071:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879566.073134:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879566.073138:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879566.073142:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879566.393080:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879566.393088:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879566.393091:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879566.393096:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879566.393101:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879566.393105:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879566.393108:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879566.393111:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879566.393114:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879567.097043:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879567.097054:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879567.097057:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879567.097061:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879567.097065:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879567.097068:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879567.097070:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879567.097072:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879567.097077:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879567.097078:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879567.097101:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879567.097105:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879567.097108:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879567.417062:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879567.417070:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879567.417072:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879567.417077:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879567.417081:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879567.417085:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879567.417088:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879567.417091:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879567.417094:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879568.121040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879568.121048:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879568.121055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879568.121056:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879568.121103:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879568.121106:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879568.121110:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879568.441035:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879568.441044:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879568.441047:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879568.441054:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879568.441059:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879568.441064:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879568.441068:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879568.441072:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879568.441075:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879569.145055:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879569.145064:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879569.145071:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879569.145073:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879569.145100:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879569.145103:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879569.145108:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879569.401048:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879569.401060:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879569.401063:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879569.401067:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879569.401072:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879569.401080:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10508 00000100:00080000:1.0:1788879569.401092:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879569.401095:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879569.401098:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879569.401107:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879569.401109:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879569.401113:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879569.401116:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879569.401118:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879569.401119:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879569.401124:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879569.401126:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879569.401130:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879569.401134:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879569.401137:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879569.401140:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879569.401142:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879569.401154:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879569.401156:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879569.401159:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879569.401162:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879569.401167:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879569.401180:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879569.401190:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100bb80 x1875765290176384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879569.401201:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879569.401206:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879569.401225:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879569.401228:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879569.401232:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879569.401234:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879569.401236:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879569.401239:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879569.401241:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10508 00000100:00080000:1.0:1788879569.401245:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879569.401247:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879569.401249:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879569.401253:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879569.401254:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879569.401257:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879569.401259:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879569.401262:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879569.401264:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879569.401268:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879569.401270:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879569.401272:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879569.401275:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1788879569.401279:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879569.401282:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1788879569.401284:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.401286:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879569.401288:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879569.401291:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:1.0:1788879569.401295:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879569.401296:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1788879569.401305:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879569.401305:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788879569.401309:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879569.401311:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879569.401316:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879569.401317:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879569.401321:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879569.401326:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1788879569.401328:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290176384:192.168.203.129@tcp:250:kworker.0 02000000:00000001:1.0:1788879569.401329:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879569.401335:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.401338:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788879569.401343:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879569.401345:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879569.401350:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879569.401355:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1788879569.401356:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100aa00 x1875765290176512/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879569.401362:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879569.401371:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879569.401373:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879569.401378:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879569.401381:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879569.401384:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879569.401385:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879569.401386:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788879569.401391:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286523264 : -117763423028352 : ffff94e51100bb80) 00000100:00080000:1.0:1788879569.401392:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879569.401400:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879569.401401:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879569.401408:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10508 00000100:00080000:1.0:1788879569.401411:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879569.401413:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879569.401414:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879569.401418:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879569.401419:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879569.401421:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879569.401424:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879569.401425:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879569.401427:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879569.401430:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879569.401434:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879569.401436:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879569.401439:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879569.401441:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879569.401444:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879569.401448:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1788879569.401449:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00000001:1.0:1788879569.401455:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879569.401459:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879569.401464:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879569.401466:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.401468:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879569.401469:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879569.401474:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879569.401477:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.401477:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879569.401484:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879569.401486:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.401488:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879569.401496:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879569.401497:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b480 x1875765290176640/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879569.401501:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305181225 00000100:00000001:1.0:1788879569.401513:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879569.401515:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.401517:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788879569.401520:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879569.401526:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879569.401530:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879569.401532:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879569.401533:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879569.401536:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879569.401539:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10508 00000100:00000001:3.0:1788879569.401541:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305181225 00000100:00080000:1.0:1788879569.401543:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879569.401550:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879569.401551:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879569.401554:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879569.401555:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:1.0:1788879569.401557:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879569.401562:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:1.0:1788879569.401566:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879569.401566:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879569.401574:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879569.401577:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879569.401578:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879569.401587:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1788879569.401588:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879569.401594:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879569.401597:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879569.401602:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788879569.401603:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879569/real 1788879569] req@ffff94e51100bb80 x1875765290176384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1788879569.401606:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879569.401612:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879569.401615:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879569.401622:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879569.401628:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879569.401636:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879569.401640:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879569.401641:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1788879569.401642:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:3.0:1788879569.401646:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1788879569.401648:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879569.401653:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.401654:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879569.401661:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879569.401663:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879569.401665:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.401667:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879569.401671:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879569.401691:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100bb80 x1875765290176384/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879569.401692:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b800 x1875765290176768/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879569.401717:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.401726:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879569.401729:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879569.401730:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.401736:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879569.401738:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879569.401739:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.401742:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879569.401746:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.401747:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879569.401755:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879569.401761:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879569.401765:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879569.401769:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290176384:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879569.401775:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.401778:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879569.401780:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879569.401783:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879569.401787:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879569.401790:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879569.401792:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879569.401797:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879569.401798:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879569.401801:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879569.401803:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.401806:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.401813:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879569.401816:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.401818:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879569.401820:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879569.401821:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879569.401823:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879569.401824:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879569.401827:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879569.401829:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879569.401831:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879569.401834:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290176512:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879569.401837:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879569.401839:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879569.401841:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879569.401843:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879569.401846:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.401853:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879569.401854:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:3.0:1788879569.401859:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879569.401874:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879569.401879:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879569.401881:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879569.401882:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.401884:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879569.401887:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.401891:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305180683 00000100:00000001:3.0:1788879569.401894:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305181225 00000100:00000001:3.0:1788879569.401897:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305180683 00000100:00000001:3.0:1788879569.401899:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305181225 00000100:00000001:3.0:1788879569.401934:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879569.401936:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.401938:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879569.401974:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879569/real 1788879569] req@ffff94e51100aa00 x1875765290176512/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879569.401985:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879569.401987:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879569.401990:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879569.401992:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879569.401994:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.401995:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879569.401997:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879569.402001:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290176512/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879569.402009:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.402012:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879569.402014:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.402016:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879569.402017:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.402019:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879569.402035:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879569.402038:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879569.402041:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879569.402043:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879569.402046:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290176512:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879569.402050:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879569.402052:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879569.402053:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879569.402055:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879569.402058:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879569.402060:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879569.402063:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290176640:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879569.402067:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879569.402069:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879569.402071:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879569.402073:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879569.402076:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.402082:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879569.402084:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:3.0:1788879569.402088:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879569.402102:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879569.402107:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879569.402109:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879569.402112:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.402114:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879569.402119:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.402124:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305180682 00000100:00000001:3.0:1788879569.402128:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305181225 00000100:00000001:3.0:1788879569.402132:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305180682 00000100:00000001:3.0:1788879569.402135:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305181225 00000100:00000001:3.0:1788879569.402138:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879569.402142:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.402145:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879569.402151:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879569/real 1788879569] req@ffff94e51100b480 x1875765290176640/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879569.402166:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879569.402168:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879569.402172:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879569.402175:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879569.402177:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.402180:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879569.402183:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879569.402190:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290176640/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879569.402208:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.402210:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879569.402213:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.402215:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879569.402216:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.402217:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879569.402233:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879569.402235:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879569.402239:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879569.402241:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879569.402244:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290176640:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879569.402248:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879569.402249:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879569.402251:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879569.402253:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879569.402255:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879569.402257:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879569.402260:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290176768:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879569.402263:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879569.402265:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879569.402267:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879569.402269:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879569.402272:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.402278:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879569.402279:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:3.0:1788879569.402283:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879569.402297:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879569.402302:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879569.402304:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879569.402305:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.402308:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879569.402310:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.402314:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305180682 00000100:00000001:3.0:1788879569.402317:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305181225 00000100:00000001:3.0:1788879569.402319:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305180682 00000100:00000001:3.0:1788879569.402321:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305181225 00000100:00000001:3.0:1788879569.402324:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879569.402325:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.402327:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879569.402331:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879569/real 1788879569] req@ffff94e51100b800 x1875765290176768/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879569.402338:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879569.402339:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879569.402341:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879569.402343:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879569.402344:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.402346:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879569.402347:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879569.402351:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290176768/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879569.402358:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.402361:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879569.402363:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.402364:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879569.402365:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.402367:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879569.402382:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879569.402384:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879569.402386:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879569.402388:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879569.402392:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290176768:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879569.402396:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.402398:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879569.402399:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879569.402401:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879569.402403:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879569.402404:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879569.402406:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879569.402408:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879569.402410:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879569.402412:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879569.402413:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.402415:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879569.402417:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879569.402418:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879569.402420:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879569.402421:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879569.402422:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879569.402424:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879569.402425:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879569.402426:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879569.402427:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.402429:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879569.402430:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879569.402431:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879569.402432:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879569.402433:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879569.402434:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879569.402436:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879569.402437:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879569.402438:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879569.402439:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.402441:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879569.402444:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879569.402445:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879569.402447:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879569.402449:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879569.465095:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879569.465108:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879569.465113:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879569.465120:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879569.465125:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879569.465131:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879569.465135:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879569.465139:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879569.465144:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879570.169016:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879570.169025:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879570.169033:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879570.169035:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879570.169072:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879570.169075:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879570.169078:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879570.489061:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879570.489070:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879570.489074:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879570.489080:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879570.489085:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879570.489090:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879570.489093:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879570.489097:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879570.489101:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879571.193081:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879571.193090:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879571.193097:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879571.193099:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879571.193179:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879571.193186:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879571.193192:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879571.513073:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879571.513082:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879571.513085:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879571.513092:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879571.513097:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879571.513102:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879571.513105:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879571.513109:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879571.513113:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879572.217151:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879572.217158:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879572.217164:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879572.217166:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879572.217203:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879572.217206:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879572.217209:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879572.537073:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879572.537080:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879572.537084:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879572.537089:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879572.537093:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879572.537096:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879572.537099:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879572.537101:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879572.537104:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879573.241047:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879573.241054:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879573.241062:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879573.241063:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879573.241098:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879573.241102:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879573.241106:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879573.561075:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879573.561083:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879573.561086:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879573.561092:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879573.561097:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879573.561102:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879573.561105:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879573.561108:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879573.561112:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879574.265084:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879574.265092:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879574.265099:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879574.265101:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879574.265128:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879574.265141:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879574.265145:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879574.521091:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879574.521104:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879574.521108:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879574.521113:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879574.521120:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879574.521128:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10513 00000100:00080000:1.0:1788879574.521143:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879574.521146:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879574.521149:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879574.521159:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879574.521162:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879574.521167:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879574.521170:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879574.521173:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879574.521175:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879574.521180:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879574.521184:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879574.521187:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879574.521192:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879574.521196:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879574.521199:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879574.521202:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879574.521213:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879574.521215:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879574.521237:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879574.521241:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879574.521247:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879574.521260:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879574.521271:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100a680 x1875765290176896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879574.521285:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879574.521291:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879574.521311:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879574.521315:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879574.521319:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879574.521321:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879574.521323:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879574.521326:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879574.521329:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10513 00000100:00080000:1.0:1788879574.521334:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879574.521337:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879574.521339:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879574.521344:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879574.521347:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879574.521350:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879574.521353:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879574.521355:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879574.521358:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879574.521362:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879574.521365:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879574.521368:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879574.521368:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879574.521378:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1788879574.521379:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879574.521386:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.521388:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879574.521389:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879574.521395:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879574.521398:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:1.0:1788879574.521402:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.521406:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879574.521408:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879574.521418:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879574.521423:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879574.521423:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788879574.521430:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879574.521432:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879574.521434:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879574.521436:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879574.521458:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879574.521459:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879574.521470:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009880 x1875765290177024/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879574.521479:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290176896:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879574.521488:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879574.521495:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.521495:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788879574.521505:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879574.521505:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1788879574.521508:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1788879574.521511:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879574.521515:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:3.0:1788879574.521518:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788879574.521519:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879574.521524:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1788879574.521527:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879574.521528:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879574.521553:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10513 00000100:00080000:1.0:1788879574.521559:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879574.521562:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879574.521564:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879574.521568:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879574.521570:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879574.521573:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:1.0:1788879574.521574:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879574.521580:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286517888 : -117763423033728 : ffff94e51100a680) 00000020:00000001:1.0:1788879574.521582:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879574.521592:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879574.521593:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879574.521595:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879574.521603:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879574.521607:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879574.521610:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879574.521614:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879574.521616:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879574.521619:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879574.521621:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879574.521624:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879574.521626:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879574.521632:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879574.521635:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879574.521637:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879574.521639:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879574.521642:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879574.521649:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879574.521655:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009180 x1875765290177152/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1788879574.521671:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879574.521674:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879574.521680:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879574.521685:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879574.521689:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1788879574.521691:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:1.0:1788879574.521695:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879574.521699:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879574.521702:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879574.521713:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879574.521716:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:3.0:1788879574.521721:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879574.521725:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10513 00000100:00000001:3.0:1788879574.521729:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:1.0:1788879574.521733:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879574.521738:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879574.521741:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879574.521744:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879574.521746:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879574.521750:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879574.521753:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:1.0:1788879574.521756:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879574.521758:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879574.521760:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879574.521768:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879574.521772:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305186345 00000100:00000001:1.0:1788879574.521773:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879574.521778:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879574.521787:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879574.521790:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879574.521792:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879574.521796:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879574.521798:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00000001:1.0:1788879574.521800:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879574.521810:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.521817:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305186345 02000000:00000001:1.0:1788879574.521818:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879574.521823:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879574.521829:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879574.521832:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879574.521844:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.521847:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879574.521852:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.521856:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788879574.521862:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008380 x1875765290177280/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879574.521865:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.521875:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1788879574.521886:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879574.521892:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879574.521893:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879574/real 1788879574] req@ffff94e51100a680 x1875765290176896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879574.521896:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879574.522028:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879574.522032:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879574.522039:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879574.522045:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879574.522048:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522053:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879574.522056:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879574.522065:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100a680 x1875765290176896/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879574.522077:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522082:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879574.522086:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522088:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879574.522089:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522092:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879574.522095:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879574.522099:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879574.522104:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879574.522107:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879574.522112:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290176896:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879574.522118:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522121:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879574.522124:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879574.522127:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879574.522133:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879574.522137:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879574.522140:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879574.522146:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879574.522149:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879574.522154:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879574.522157:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522161:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522172:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879574.522176:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.522180:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879574.522183:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879574.522187:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879574.522190:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879574.522193:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879574.522199:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879574.522203:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879574.522208:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879574.522232:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290177024:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879574.522240:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879574.522244:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879574.522247:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879574.522250:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879574.522256:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.522265:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879574.522267:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286514304 : -117763423037312 : ffff94e511009880) 00000100:00000001:3.0:1788879574.522273:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879574.522293:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879574.522300:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879574.522303:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879574.522305:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.522307:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879574.522311:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.522316:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305185804 00000100:00000001:3.0:1788879574.522320:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305186345 00000100:00000001:3.0:1788879574.522323:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305185804 00000100:00000001:3.0:1788879574.522326:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305186345 00000100:00000001:3.0:1788879574.522328:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879574.522331:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.522333:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879574.522338:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879574/real 1788879574] req@ffff94e511009880 x1875765290177024/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879574.522348:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879574.522349:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879574.522352:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879574.522354:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879574.522356:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522358:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879574.522360:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879574.522365:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009880 x1875765290177024/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879574.522375:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522377:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879574.522380:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522382:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879574.522383:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522386:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879574.522410:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879574.522416:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879574.522422:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879574.522426:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879574.522433:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290177024:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879574.522441:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879574.522444:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879574.522447:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879574.522461:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879574.522464:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879574.522469:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879574.522475:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290177152:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879574.522482:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879574.522485:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879574.522489:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879574.522491:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879574.522495:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.522504:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879574.522505:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286512512 : -117763423039104 : ffff94e511009180) 00000100:00000001:3.0:1788879574.522510:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879574.522528:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879574.522534:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879574.522537:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879574.522540:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.522542:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879574.522545:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.522569:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305185802 00000100:00000001:3.0:1788879574.522573:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305186345 00000100:00000001:3.0:1788879574.522576:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305185802 00000100:00000001:3.0:1788879574.522579:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305186345 00000100:00000001:3.0:1788879574.522582:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879574.522584:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.522586:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879574.522592:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879574/real 1788879574] req@ffff94e511009180 x1875765290177152/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879574.522602:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879574.522603:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879574.522606:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879574.522608:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879574.522610:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522613:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879574.522615:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879574.522619:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009180 x1875765290177152/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879574.522627:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522630:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879574.522633:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522634:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879574.522636:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522638:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879574.522656:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879574.522659:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879574.522663:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879574.522666:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879574.522670:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290177152:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879574.522675:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879574.522677:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879574.522679:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879574.522681:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879574.522684:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879574.522686:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879574.522689:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290177280:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879574.522693:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879574.522695:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879574.522698:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879574.522700:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879574.522703:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.522719:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879574.522721:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508928 : -117763423042688 : ffff94e511008380) 00000100:00000001:3.0:1788879574.522726:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879574.522746:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879574.522753:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879574.522758:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879574.522760:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.522763:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879574.522768:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.522775:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305185802 00000100:00000001:3.0:1788879574.522780:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305186345 00000100:00000001:3.0:1788879574.522785:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305185802 00000100:00000001:3.0:1788879574.522789:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305186345 00000100:00000001:3.0:1788879574.522793:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879574.522798:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.522802:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879574.522811:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879574/real 1788879574] req@ffff94e511008380 x1875765290177280/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879574.522852:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879574.522854:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879574.522858:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879574.522861:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879574.522862:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522865:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879574.522867:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879574.522873:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008380 x1875765290177280/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879574.522881:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522885:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879574.522888:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522890:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879574.522891:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522894:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879574.522976:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879574.522981:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879574.522985:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879574.522987:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879574.522991:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290177280:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879574.522995:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.522998:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879574.523007:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879574.523009:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879574.523030:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879574.523032:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879574.523034:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879574.523038:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879574.523040:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879574.523043:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879574.523045:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.523049:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879574.523050:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879574.523052:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879574.523054:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879574.523056:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879574.523058:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879574.523060:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879574.523061:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879574.523063:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879574.523065:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.523067:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879574.523068:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879574.523069:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879574.523071:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879574.523072:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879574.523074:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879574.523075:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879574.523077:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879574.523079:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879574.523080:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.523082:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879574.523087:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879574.523089:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879574.523091:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879574.523093:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879574.585047:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879574.585055:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879574.585058:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879574.585065:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879574.585069:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879574.585074:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879574.585078:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879574.585081:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879574.585085:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879575.289088:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879575.289098:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879575.289109:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879575.289115:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879575.289118:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879575.289122:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879575.289126:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879575.289129:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879575.289132:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879575.289135:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879575.289177:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879575.289180:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879575.289184:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879575.609039:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879575.609047:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879575.609049:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879575.609056:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879575.609060:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879575.609064:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879575.609067:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879575.609070:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879575.609073:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879576.313009:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879576.313018:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879576.313026:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879576.313028:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879576.313074:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879576.313079:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879576.313085:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879576.632986:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879576.632994:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879576.632996:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879576.633001:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879576.633004:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879576.633008:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879576.633011:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879576.633013:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879576.633016:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879577.337041:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879577.337048:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879577.337051:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879577.337054:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879577.337060:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879577.337062:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879577.337064:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879577.337067:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879577.337071:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879577.337072:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879577.337135:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879577.337138:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879577.337145:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879577.657059:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879577.657069:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879577.657072:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879577.657081:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879577.657086:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879577.657092:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879577.657097:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879577.657100:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879577.657104:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879578.361069:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879578.361077:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879578.361084:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879578.361086:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879578.361114:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879578.361118:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879578.361122:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879578.681003:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879578.681011:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879578.681013:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879578.681018:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879578.681021:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879578.681025:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879578.681028:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879578.681030:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879578.681032:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879579.385077:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879579.385085:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879579.385093:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879579.385095:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879579.385135:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879579.385138:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879579.385141:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879579.641083:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879579.641095:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879579.641099:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879579.641103:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879579.641109:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879579.641118:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10518 00000100:00080000:0.0:1788879579.641131:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879579.641134:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879579.641137:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879579.641146:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879579.641150:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879579.641154:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879579.641157:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879579.641159:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879579.641161:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879579.641167:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879579.641171:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879579.641174:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879579.641179:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879579.641200:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.641203:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879579.641206:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879579.641220:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879579.641222:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.641225:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879579.641229:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879579.641235:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.641268:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879579.641280:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290177408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879579.641309:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879579.641332:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.641348:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879579.641352:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879579.641356:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879579.641358:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879579.641360:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879579.641363:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879579.641366:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10518 00000100:00080000:0.0:1788879579.641370:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879579.641373:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879579.641374:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879579.641378:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879579.641397:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879579.641399:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879579.641402:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879579.641404:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879579.641407:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879579.641411:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879579.641414:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879579.641416:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879579.641423:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879579.641425:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.641428:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879579.641432:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879579.641436:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.641439:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879579.641444:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879579.641447:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.641449:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879579.641451:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879579.641454:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.641461:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.641470:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879579.641478:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1788879579.641481:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979180 x1875765290177536/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879579.641483:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879579.641488:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879579.641493:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879579.641523:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879579.641529:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879579.641533:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788879579.641536:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.641544:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879579.641549:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290177408:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879579.641556:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.641559:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1788879579.641582:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1788879579.641585:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879579.641589:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879579.641593:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:3.0:1788879579.641595:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879579.641604:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:3.0:1788879579.641606:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879579.641608:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879579.641617:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879579.641622:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10518 00000100:00080000:0.0:1788879579.641646:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879579.641649:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879579.641652:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879579.641653:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879579.641662:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000020:00000001:0.0:1788879579.641664:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879579.641670:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879579.641671:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1788879579.641676:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879579.641682:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879579.641686:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879579.641689:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879579.641693:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879579.641697:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879579.641699:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879579.641705:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879579.641708:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.641728:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879579.641732:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879579.641746:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.641749:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879579.641756:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879579.641758:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.641765:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000100:3.0:1788879579.641767:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:0.0:1788879579.641768:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879579.641775:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.641783:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.641789:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00100000:0.0:1788879579.641792:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290177664/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879579.641798:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879579.641805:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.641830:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879579.641832:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879579.641843:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.641846:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.641855:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.641857:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305191465 00000100:00080000:0.0:1788879579.641859:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879579.641871:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879579.641873:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879579.641953:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879579.641960:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879579.641966:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10518 00000100:00080000:0.0:1788879579.641974:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879579.641979:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879579.641982:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788879579.641983:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879579.641996:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879579.642000:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879579.642001:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305191466 00000020:00000001:0.0:1788879579.642005:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879579.642015:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879579.642015:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788879579.642025:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879579.642030:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879579.642035:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1788879579.642041:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879579.642043:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642052:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879579.642053:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879579.642059:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879579.642064:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788879579.642068:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879579/real 1788879579] req@ffff94e50897b800 x1875765290177408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788879579.642070:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.642078:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879579.642082:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879579.642090:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.642097:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879579.642107:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1788879579.642108:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000400:00000001:3.0:1788879579.642114:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1788879579.642115:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.642122:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879579.642124:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879579.642126:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:0.0:1788879579.642127:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642134:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879579.642139:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642140:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642149:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879579.642155:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879579.642156:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290177792/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1788879579.642167:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290177408/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879579.642175:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879579.642182:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.642189:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642192:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642199:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879579.642203:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642206:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879579.642207:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642210:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879579.642213:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879579.642218:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879579.642223:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879579.642226:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879579.642230:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290177408:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879579.642236:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642239:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879579.642241:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879579.642244:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879579.642248:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879579.642250:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879579.642253:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879579.642257:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879579.642259:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879579.642263:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879579.642265:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642267:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642276:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879579.642278:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642281:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879579.642283:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879579.642285:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879579.642287:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879579.642289:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879579.642291:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879579.642293:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879579.642296:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879579.642299:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290177536:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879579.642303:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879579.642306:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879579.642308:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879579.642310:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879579.642313:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642321:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879579.642322:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 00000100:00000001:3.0:1788879579.642328:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879579.642346:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879579.642351:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879579.642354:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879579.642375:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642377:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879579.642382:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642387:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305190924 00000100:00000001:3.0:1788879579.642391:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305191465 00000100:00000001:3.0:1788879579.642394:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305190924 00000100:00000001:3.0:1788879579.642396:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305191466 00000100:00000001:3.0:1788879579.642400:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879579.642402:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642404:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879579.642409:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879579/real 1788879579] req@ffff94e508979180 x1875765290177536/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879579.642419:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879579.642420:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879579.642422:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879579.642425:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879579.642427:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642429:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879579.642431:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879579.642435:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290177536/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879579.642443:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642446:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879579.642448:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642450:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879579.642452:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642453:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879579.642472:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879579.642475:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879579.642479:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879579.642482:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879579.642486:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290177536:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879579.642490:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879579.642492:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879579.642494:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879579.642496:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879579.642498:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879579.642500:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879579.642503:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290177664:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879579.642507:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879579.642509:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879579.642511:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879579.642531:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879579.642535:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642542:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879579.642544:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 00000100:00000001:3.0:1788879579.642549:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879579.642563:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879579.642568:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879579.642571:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879579.642572:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642575:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879579.642578:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642599:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305190922 00000100:00000001:3.0:1788879579.642602:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305191465 00000100:00000001:3.0:1788879579.642605:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305190922 00000100:00000001:3.0:1788879579.642607:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305191466 00000100:00000001:3.0:1788879579.642609:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879579.642611:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642613:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879579.642618:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879579/real 1788879579] req@ffff94e508978380 x1875765290177664/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879579.642627:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879579.642629:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879579.642631:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879579.642633:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879579.642634:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642636:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879579.642638:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879579.642642:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290177664/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879579.642650:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642652:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879579.642654:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642656:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879579.642657:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642659:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879579.642676:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879579.642678:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879579.642681:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879579.642683:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879579.642686:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290177664:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879579.642704:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879579.642706:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879579.642707:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879579.642710:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879579.642712:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879579.642719:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879579.642722:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290177792:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879579.642726:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879579.642729:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879579.642730:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879579.642733:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879579.642736:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642741:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879579.642743:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:3.0:1788879579.642746:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879579.642759:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879579.642763:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879579.642766:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879579.642767:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642769:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879579.642772:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642775:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305190922 00000100:00000001:3.0:1788879579.642778:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305191465 00000100:00000001:3.0:1788879579.642781:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305190922 00000100:00000001:3.0:1788879579.642783:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305191466 00000100:00000001:3.0:1788879579.642786:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879579.642788:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.642790:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879579.642794:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879579/real 1788879579] req@ffff94e50897bb80 x1875765290177792/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879579.642802:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879579.642803:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879579.642805:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879579.642807:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879579.642809:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642811:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879579.642812:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879579.642816:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290177792/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879579.642824:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642826:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879579.642829:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642831:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879579.642832:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642834:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879579.642855:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879579.642860:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879579.642865:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879579.642869:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879579.642894:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290177792:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879579.642931:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642935:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879579.642938:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879579.642941:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879579.642945:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879579.642947:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879579.642949:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879579.642970:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879579.642973:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879579.642977:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879579.642978:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642981:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879579.642982:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879579.642984:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879579.642987:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879579.642988:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879579.642989:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879579.642991:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879579.642993:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879579.642995:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879579.642996:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.642998:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879579.642999:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879579.643001:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879579.643003:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879579.643004:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879579.643005:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879579.643007:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879579.643010:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879579.643012:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879579.643014:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.643016:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879579.643021:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879579.643023:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879579.643025:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879579.643028:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879579.705086:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879579.705097:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879579.705101:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879579.705108:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879579.705114:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879579.705122:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879579.705133:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879579.705139:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879579.705144:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879580.409061:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879580.409071:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879580.409077:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879580.409079:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879580.409107:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879580.409111:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879580.409114:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879580.729016:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879580.729033:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879580.729037:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879580.729048:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879580.729056:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879580.729063:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879580.729068:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879580.729074:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879580.729080:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879581.433029:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879581.433035:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879581.433040:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879581.433041:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879581.433085:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879581.433087:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879581.433090:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879581.752998:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879581.753006:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879581.753008:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879581.753013:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879581.753028:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879581.753032:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879581.753034:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879581.753037:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879581.753039:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879582.457081:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879582.457090:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879582.457098:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879582.457100:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879582.457144:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879582.457147:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879582.457151:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879582.777033:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879582.777042:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879582.777045:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879582.777052:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879582.777057:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879582.777062:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879582.777065:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879582.777068:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879582.777072:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879583.481013:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879583.481021:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879583.481028:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879583.481030:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879583.481057:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879583.481060:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879583.481065:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879583.801074:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879583.801085:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879583.801090:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879583.801098:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879583.801105:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879583.801112:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879583.801117:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879583.801122:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879583.801128:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879584.505054:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879584.505081:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879584.505089:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879584.505092:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879584.505125:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879584.505128:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879584.505132:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879584.761076:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879584.761088:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879584.761091:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879584.761096:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879584.761102:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879584.761112:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10523 00000100:00080000:2.0:1788879584.761126:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879584.761129:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879584.761131:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879584.761141:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879584.761144:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879584.761148:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879584.761151:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879584.761153:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879584.761155:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879584.761160:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879584.761163:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879584.761165:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879584.761170:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879584.761173:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879584.761175:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879584.761178:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879584.761194:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879584.761195:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879584.761198:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879584.761201:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879584.761206:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879584.761221:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879584.761248:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5f80 x1875765290177920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879584.761265:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879584.761272:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879584.761300:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879584.761306:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879584.761312:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879584.761316:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879584.761320:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879584.761326:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879584.761331:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10523 00000100:00000001:3.0:1788879584.761334:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788879584.761337:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879584.761343:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879584.761344:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879584.761351:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879584.761352:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879584.761361:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1788879584.761362:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879584.761367:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879584.761368:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879584.761373:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879584.761379:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:3.0:1788879584.761380:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879584.761389:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1788879584.761390:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879584.761392:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:3.0:1788879584.761402:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879584.761406:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879584.761411:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879584.761418:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879584.761428:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788879584.761428:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290177920:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879584.761446:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.761448:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788879584.761455:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:3.0:1788879584.761457:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879584.761463:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879584.761467:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:2.0:1788879584.761469:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1788879584.761481:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788879584.761482:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879584.761503:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879584.761511:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879584.761513:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879584.761516:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879584.761518:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879584.761518:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788879584.761523:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.761524:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074556288 : -117763634995328 : ffff94e5045e5f80) 00000100:00000001:3.0:1788879584.761538:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879584.761542:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879584.761552:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290178048/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879584.761580:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879584.761583:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879584.761590:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879584.761593:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879584.761598:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879584.761600:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879584.761603:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879584.761607:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879584.761616:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10523 00000800:00000100:3.0:1788879584.761623:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788879584.761627:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879584.761633:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879584.761635:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879584.761642:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879584.761645:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879584.761646:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:2.0:1788879584.761651:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879584.761655:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879584.761658:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879584.761662:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:3.0:1788879584.761666:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788879584.761672:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879584.761674:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.761680:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879584.761688:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.761693:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305196585 00000100:00000001:2.0:1788879584.761713:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879584.761719:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1788879584.761719:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1788879584.761728:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879584.761731:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305196585 02000000:00000001:2.0:1788879584.761736:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.761740:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879584.761741:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879584.761748:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879584.761751:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1788879584.761756:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.761759:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879584.761761:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.761766:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1788879584.761773:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879584.761779:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879584.761781:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879584/real 1788879584] req@ffff94e5045e5f80 x1875765290177920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879584.761784:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879584.761788:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879584.761795:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879584.761808:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1788879584.761812:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879584.761818:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:2.0:1788879584.761820:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290178176/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879584.761825:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879584.761834:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879584.761856:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879584.761860:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879584.761862:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879584.761866:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879584.761885:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879584.761886:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5f80 x1875765290177920/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879584.761900:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879584.761907:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879584.761914:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879584.761917:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1788879584.761939:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.761944:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879584.761951:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.761955:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879584.761958:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.761962:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879584.761966:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879584.761968:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879584.761981:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879584.761982:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879584.761992:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10523 00000100:00080000:3.0:1788879584.762003:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00080000:2.0:1788879584.762006:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879584.762011:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788879584.762013:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879584.762017:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:3.0:1788879584.762024:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290177920:192.168.203.129@tcp:250:kworker.0 00000020:00000001:2.0:1788879584.762028:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1788879584.762032:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788879584.762033:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1788879584.762038:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1788879584.762039:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879584.762043:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000020:00000001:2.0:1788879584.762045:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:3.0:1788879584.762048:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788879584.762050:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879584.762054:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:3.0:1788879584.762055:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879584.762061:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00080000:2.0:1788879584.762062:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000020:00000001:3.0:1788879584.762069:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:2.0:1788879584.762070:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879584.762075:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879584.762079:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879584.762079:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879584.762087:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1788879584.762088:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879584.762093:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879584.762096:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000008:00100000:2.0:1788879584.762097:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1788879584.762100:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1788879584.762110:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762110:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879584.762132:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762138:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1788879584.762140:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879584.762144:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762144:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879584.762161:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1788879584.762163:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879584.762166:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1788879584.762169:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788879584.762172:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762173:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879584.762179:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879584.762185:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762185:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879584.762199:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:2.0:1788879584.762205:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4380 x1875765290178304/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1788879584.762205:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879584.762220:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879584.762236:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290178048:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879584.762249:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788879584.762250:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1788879584.762253:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879584.762256:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879584.762258:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879584.762260:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879584.762268:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879584.762273:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762280:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879584.762283:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 00000100:00000001:3.0:1788879584.762288:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879584.762312:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879584.762318:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879584.762321:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879584.762323:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762324:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879584.762327:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762332:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305196044 00000100:00000001:3.0:1788879584.762335:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305196585 00000100:00000001:3.0:1788879584.762338:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305196044 00000100:00000001:3.0:1788879584.762340:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305196585 00000100:00000001:3.0:1788879584.762342:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879584.762344:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762364:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879584.762370:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879584/real 1788879584] req@ffff94e5045e6300 x1875765290178048/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879584.762381:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879584.762382:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879584.762385:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879584.762388:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879584.762390:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762392:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879584.762394:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879584.762398:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290178048/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879584.762407:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762409:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879584.762412:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762414:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879584.762415:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762417:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879584.762452:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879584.762455:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879584.762458:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879584.762460:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879584.762463:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290178048:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879584.762467:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879584.762469:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879584.762471:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879584.762473:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879584.762475:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879584.762478:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879584.762481:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290178176:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879584.762487:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879584.762489:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879584.762491:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879584.762492:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879584.762496:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762502:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879584.762503:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000100:00000001:3.0:1788879584.762507:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879584.762518:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879584.762522:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879584.762524:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879584.762526:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762528:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879584.762530:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762534:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305196042 00000100:00000001:3.0:1788879584.762536:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305196585 00000100:00000001:3.0:1788879584.762539:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305196042 00000100:00000001:3.0:1788879584.762541:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305196585 00000100:00000001:3.0:1788879584.762544:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879584.762547:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762549:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879584.762556:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879584/real 1788879584] req@ffff94e5045e7800 x1875765290178176/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879584.762568:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879584.762570:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879584.762575:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879584.762577:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879584.762580:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762582:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879584.762585:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879584.762592:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290178176/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879584.762615:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762619:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879584.762623:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762625:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879584.762628:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762631:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879584.762647:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879584.762650:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879584.762654:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879584.762656:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879584.762660:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290178176:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879584.762664:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762666:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879584.762668:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879584.762670:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879584.762673:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879584.762674:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879584.762676:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879584.762679:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879584.762681:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879584.762683:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879584.762684:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762686:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879584.762687:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879584.762689:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879584.762691:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879584.762692:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879584.762693:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879584.762694:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879584.762695:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879584.762697:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879584.762698:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762700:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762703:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879584.762706:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762708:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879584.762710:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879584.762711:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879584.762713:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879584.762714:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879584.762717:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879584.762719:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879584.762721:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879584.762724:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290178304:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879584.762728:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879584.762730:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879584.762732:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879584.762734:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879584.762737:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762744:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879584.762745:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074549120 : -117763635002496 : ffff94e5045e4380) 00000100:00000001:3.0:1788879584.762750:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1788879584.762763:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879584.762767:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879584.762770:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879584.762771:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762773:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879584.762776:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762779:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305196042 00000100:00000001:3.0:1788879584.762782:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305196585 00000100:00000001:3.0:1788879584.762785:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305196042 00000100:00000001:3.0:1788879584.762787:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305196585 00000100:00000001:3.0:1788879584.762789:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879584.762791:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762793:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879584.762797:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879584/real 1788879584] req@ffff94e5045e4380 x1875765290178304/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1788879584.762811:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1788879584.762812:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879584.762815:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879584.762817:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879584.762818:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762819:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879584.762821:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879584.762825:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4380 x1875765290178304/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879584.762832:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762834:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879584.762837:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762858:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879584.762861:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762863:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879584.762878:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879584.762881:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879584.762884:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879584.762887:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1788879584.762890:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290178304:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879584.762895:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762897:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879584.762898:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1788879584.762900:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1788879584.762903:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879584.762928:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879584.762929:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879584.762932:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879584.762933:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879584.762936:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1788879584.762937:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762940:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879584.762943:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879584.762945:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879584.762947:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879584.762949:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879584.825050:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879584.825061:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879584.825065:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879584.825073:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879584.825077:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879584.825082:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879584.825085:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879584.825088:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879584.825092:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879585.529045:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879585.529056:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879585.529066:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879585.529071:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879585.529075:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879585.529082:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879585.529117:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879585.529121:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879585.529125:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879585.529130:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879585.529163:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879585.529167:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879585.529171:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879585.849049:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879585.849058:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879585.849061:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879585.849068:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879585.849072:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879585.849078:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879585.849082:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879585.849085:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879585.849090:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879586.553006:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879586.553013:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879586.553019:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879586.553021:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879586.553042:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879586.553045:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879586.553049:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879586.873034:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879586.873045:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879586.873049:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879586.873058:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879586.873064:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879586.873070:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879586.873075:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879586.873080:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879586.873084:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879587.577028:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879587.577038:0:178131:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879587.577041:0:178131:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879587.577045:0:178131:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879587.577051:0:178131:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879587.577053:0:178131:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879587.577056:0:178131:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879587.577059:0:178131:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879587.577063:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879587.577065:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879587.577090:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879587.577094:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879587.577099:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879587.897044:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879587.897052:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879587.897055:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879587.897060:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879587.897064:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879587.897069:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879587.897072:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879587.897075:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879587.897078:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879588.601033:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879588.601040:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879588.601046:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879588.601048:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879588.601085:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879588.601088:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879588.601091:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879588.921005:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879588.921013:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879588.921016:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879588.921021:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879588.921025:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879588.921030:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879588.921032:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879588.921035:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879588.921038:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879589.625045:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879589.625055:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879589.625063:0:178131:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879589.625066:0:178131:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879589.625097:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879589.625102:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879589.625108:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879589.880999:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879589.881011:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879589.881014:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879589.881018:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879589.881024:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879589.881034:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10528 00000100:00080000:2.0:1788879589.881063:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879589.881069:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879589.881075:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879589.881087:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879589.881091:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879589.881097:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879589.881102:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879589.881107:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879589.881110:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879589.881118:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879589.881122:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879589.881126:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879589.881131:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879589.881136:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881139:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879589.881143:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879589.881165:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879589.881167:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881171:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879589.881175:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879589.881180:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881194:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879589.881206:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290178432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879589.881220:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879589.881225:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881266:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879589.881269:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879589.881273:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879589.881274:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879589.881276:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879589.881279:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879589.881282:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10528 00000100:00080000:2.0:1788879589.881287:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879589.881289:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879589.881291:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879589.881296:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879589.881297:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879589.881301:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879589.881303:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879589.881306:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879589.881308:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879589.881312:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879589.881315:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879589.881317:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879589.881321:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879589.881324:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879589.881328:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.881329:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879589.881331:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879589.881336:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:2.0:1788879589.881338:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879589.881343:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:2.0:1788879589.881343:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881349:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.881350:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879589.881355:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879589.881358:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879589.881359:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881364:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879589.881366:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879589.881367:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879589.881371:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879589.881375:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881379:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879589.881388:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290178432:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1788879589.881389:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290178560/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879589.881401:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879589.881406:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788879589.881413:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879589.881414:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881418:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881427:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879589.881434:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1788879589.881435:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879589.881443:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:1.0:1788879589.881444:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881446:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879589.881451:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879589.881456:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879589.881462:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10528 00000100:00000001:1.0:1788879589.881469:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788879589.881469:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879589.881475:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:2.0:1788879589.881477:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879589.881481:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879589.881485:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1788879589.881488:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879589.881493:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879589.881496:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879589.881498:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879589.881501:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879589.881502:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879589.881506:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879589.881509:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879589.881511:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879589.881515:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879589.881518:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881521:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879589.881524:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1788879589.881536:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879589.881552:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879589.881558:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879589.881562:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.881566:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000008:00000001:2.0:1788879589.881571:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881575:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.881579:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879589.881581:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879589.881586:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.881590:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305201705 00000100:00000001:2.0:1788879589.881590:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879589.881594:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879589.881599:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881624:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.881627:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00100000:2.0:1788879589.881636:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290178688/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879589.881643:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305201705 00000100:00000001:1.0:1788879589.881658:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879589.881660:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879589.881668:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.881670:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879589.881677:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.881679:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.881690:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1788879589.881690:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879589.881702:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879589.881708:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788879589.881710:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879589/real 1788879589] req@ffff94e5045e5500 x1875765290178432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879589.881713:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879589.881732:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879589.881738:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10528 00000100:00080000:2.0:1788879589.881744:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879589.881746:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879589.881748:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879589.881754:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879589.881758:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879589.881764:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879589.881770:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879589.881775:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879589.881777:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:1.0:1788879589.881780:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1788879589.881783:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000400:00000001:1.0:1788879589.881787:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879589.881788:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879589.881795:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879589.881799:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879589.881801:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:2.0:1788879589.881824:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881829:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879589.881831:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000008:00100000:2.0:1788879589.881833:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879589.881856:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1788879589.881856:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881873:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.881875:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879589.881881:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879589.881881:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879589.881887:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881892:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1788879589.881894:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290178432/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879589.881896:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879589.881901:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879589.881930:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879589.881984:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290178816/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879589.881987:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.882003:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879589.882004:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879589.882011:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.882016:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879589.882018:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.882029:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879589.882034:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.882040:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879589.882046:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879589.882053:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879589.882064:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879589.882072:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879589.882081:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290178432:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879589.882092:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.882097:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879589.882101:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879589.882106:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879589.882113:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879589.882118:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879589.882121:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879589.882127:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879589.882130:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879589.882134:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879589.882137:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.882142:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.882164:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879589.882169:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.882172:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879589.882175:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879589.882177:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879589.882180:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879589.882183:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879589.882187:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879589.882190:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879589.882193:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879589.882201:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290178560:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879589.882207:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879589.882211:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879589.882214:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879589.882218:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879589.882223:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.882234:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879589.882236:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:1.0:1788879589.882260:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879589.882281:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879589.882288:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879589.882292:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879589.882294:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.882296:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879589.882300:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.882306:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305201164 00000100:00000001:1.0:1788879589.882310:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305201705 00000100:00000001:1.0:1788879589.882314:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305201164 00000100:00000001:1.0:1788879589.882317:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305201705 00000100:00000001:1.0:1788879589.882320:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879589.882323:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.882326:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879589.882332:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879589/real 1788879589] req@ffff94e5045e5c00 x1875765290178560/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879589.882346:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879589.882348:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879589.882352:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879589.882355:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879589.882356:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.882359:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879589.882361:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879589.882367:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290178560/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879589.882378:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.882382:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879589.882385:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.882387:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879589.882389:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.882391:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879589.882425:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879589.882429:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879589.882434:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879589.882438:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879589.882442:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290178560:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879589.882449:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879589.882451:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879589.882453:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879589.882456:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879589.882459:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879589.882462:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879589.882466:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290178688:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879589.882471:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879589.882473:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879589.882476:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879589.882478:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879589.882482:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.882491:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879589.882493:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:1.0:1788879589.882497:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879589.882534:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879589.882543:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879589.882547:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879589.882550:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.882553:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879589.882558:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.882564:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305201162 00000100:00000001:1.0:1788879589.882569:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305201705 00000100:00000001:1.0:1788879589.882573:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305201162 00000100:00000001:1.0:1788879589.882576:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305201705 00000100:00000001:1.0:1788879589.882580:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879589.882583:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.882587:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879589.882594:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879589/real 1788879589] req@ffff94e5045e7100 x1875765290178688/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879589.882609:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879589.882611:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879589.882616:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879589.882619:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879589.882622:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.882626:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879589.882628:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879589.882636:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290178688/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879589.882650:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.882654:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879589.882659:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.882662:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879589.882664:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.882668:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879589.882690:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879589.882696:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879589.882702:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879589.882706:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879589.882712:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290178688:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879589.882719:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879589.882731:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879589.882734:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879589.882737:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879589.882741:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879589.882745:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879589.882751:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290178816:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879589.882757:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879589.882760:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879589.882763:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879589.882766:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879589.882771:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.882781:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879589.882784:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:1.0:1788879589.882789:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879589.882854:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879589.882862:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879589.882865:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879589.882868:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.882870:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879589.882874:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.882879:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305201162 00000100:00000001:1.0:1788879589.882883:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305201705 00000100:00000001:1.0:1788879589.882886:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305201162 00000100:00000001:1.0:1788879589.882889:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305201705 00000100:00000001:1.0:1788879589.882909:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879589.882913:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.882916:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879589.882972:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879589/real 1788879589] req@ffff94e5045e4a80 x1875765290178816/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879589.882989:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879589.882992:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879589.882997:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879589.883000:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879589.883002:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.883005:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879589.883008:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879589.883015:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290178816/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879589.883027:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.883032:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879589.883035:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.883038:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879589.883040:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.883043:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879589.883065:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879589.883070:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879589.883077:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879589.883081:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879589.883086:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290178816:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879589.883092:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.883096:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879589.883098:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879589.883100:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879589.883105:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879589.883107:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879589.883109:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879589.883113:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879589.883115:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879589.883118:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879589.883120:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.883123:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879589.883125:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879589.883127:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879589.883140:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879589.883142:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879589.883144:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879589.883147:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879589.883149:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879589.883151:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879589.883153:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.883156:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879589.883158:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879589.883159:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879589.883162:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879589.883164:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879589.883165:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879589.883169:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879589.883170:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879589.883173:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879589.883175:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.883178:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879589.883185:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879589.883187:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879589.883191:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879589.883193:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879589.945020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879589.945029:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879589.945032:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879589.945038:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879589.945042:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879589.945047:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879589.945051:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879589.945055:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879589.945058:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879590.649020:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879590.649026:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879590.649032:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879590.649033:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879590.649052:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879590.649055:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879590.649057:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879590.969052:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879590.969060:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879590.969063:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879590.969071:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879590.969076:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879590.969081:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879590.969085:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879590.969089:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879590.969093:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879591.673068:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879591.673077:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879591.673085:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879591.673088:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879591.673115:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879591.673118:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879591.673121:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879591.993009:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879591.993017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879591.993020:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879591.993026:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879591.993032:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879591.993037:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879591.993042:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879591.993047:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879591.993053:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879592.697044:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879592.697052:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879592.697058:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879592.697060:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879592.697084:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879592.697087:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879592.697091:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879593.017052:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879593.017061:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879593.017064:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879593.017071:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879593.017075:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879593.017080:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879593.017084:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879593.017088:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879593.017091:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879593.721022:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879593.721029:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879593.721037:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879593.721039:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879593.721058:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879593.721060:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879593.721062:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1788879594.040996:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879594.041006:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879594.041009:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879594.041015:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879594.041020:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879594.041025:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879594.041028:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879594.041032:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879594.041045:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879594.744977:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879594.744981:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879594.744986:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879594.744987:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879594.745002:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879594.745004:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879594.745006:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879595.001040:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879595.001051:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879595.001065:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879595.001069:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879595.001074:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879595.001082:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10533 00000100:00080000:2.0:1788879595.001094:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879595.001098:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879595.001100:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879595.001109:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879595.001112:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879595.001116:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879595.001119:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879595.001121:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879595.001123:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879595.001143:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879595.001146:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879595.001149:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879595.001153:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879595.001156:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001158:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879595.001161:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879595.001174:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879595.001175:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001179:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879595.001182:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879595.001187:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001199:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879595.001211:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290178944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879595.001222:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879595.001227:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001246:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879595.001249:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879595.001253:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879595.001254:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879595.001256:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879595.001259:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879595.001261:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10533 00000100:00080000:2.0:1788879595.001264:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879595.001266:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879595.001267:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879595.001270:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879595.001272:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879595.001274:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879595.001276:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879595.001278:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879595.001279:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879595.001282:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879595.001284:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879595.001285:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879595.001291:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879595.001294:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001295:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879595.001300:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879595.001303:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001305:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879595.001309:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879595.001311:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001312:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879595.001314:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879595.001317:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001322:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879595.001329:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290179072/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879595.001337:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879595.001339:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001352:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879595.001355:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879595.001360:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879595.001363:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879595.001366:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1788879595.001366:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879595.001377:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879595.001378:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879595.001381:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879595.001384:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1788879595.001389:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10533 00000100:00000001:1.0:1788879595.001390:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879595.001400:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788879595.001406:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879595.001407:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879595.001411:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879595.001413:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879595.001417:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879595.001427:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:1.0:1788879595.001431:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290178944:192.168.203.129@tcp:250:kworker.0 00000020:00000001:2.0:1788879595.001432:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879595.001444:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879595.001445:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879595.001451:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879595.001454:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879595.001454:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879595.001469:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879595.001484:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879595.001486:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1788879595.001493:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879595.001497:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879595.001499:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879595.001500:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879595.001510:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879595.001515:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001518:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879595.001521:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879595.001525:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001527:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.001530:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788879595.001533:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879595.001535:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 02000000:00000001:2.0:1788879595.001537:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001546:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879595.001547:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1788879595.001555:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879595.001559:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001566:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879595.001574:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290179200/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879595.001583:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879595.001585:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001590:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879595.001593:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879595.001597:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879595.001600:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879595.001602:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879595.001606:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000800:00000100:1.0:1788879595.001613:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788879595.001615:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10533 00000100:00080000:2.0:1788879595.001623:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879595.001625:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879595.001628:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879595.001634:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879595.001635:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:2.0:1788879595.001636:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879595.001643:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879595.001647:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879595.001648:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879595.001653:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001655:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879595.001662:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879595.001664:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879595.001674:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879595.001675:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879595.001686:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.001688:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305206825 02000000:00000001:2.0:1788879595.001692:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879595.001727:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879595.001731:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001734:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879595.001735:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00100000:2.0:1788879595.001738:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879595.001744:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879595.001746:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.001750:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305206825 02000000:00000001:2.0:1788879595.001753:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879595.001757:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.001759:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879595.001762:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879595.001787:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879595.001818:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879595.001822:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879595.001823:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.001827:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879595.001833:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879595.001839:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879595/real 1788879595] req@ffff94e5045e5180 x1875765290178944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879595.001847:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290179328/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1788879595.001881:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788879595.001883:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:1.0:1788879595.001884:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879595.001889:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.001891:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879595.001894:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.001896:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879595.001899:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.001902:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879595.001904:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879595.001911:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290178944/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879595.001990:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.001996:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879595.001999:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002003:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879595.002004:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002008:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879595.002011:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879595.002014:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879595.002021:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879595.002024:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879595.002028:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290178944:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879595.002032:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879595.002034:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879595.002036:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879595.002038:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879595.002041:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879595.002043:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879595.002046:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290179072:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879595.002049:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879595.002052:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879595.002054:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879595.002055:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879595.002059:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.002067:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879595.002068:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:1.0:1788879595.002074:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879595.002106:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879595.002111:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879595.002114:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879595.002115:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.002117:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879595.002120:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.002137:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305206284 00000100:00000001:1.0:1788879595.002140:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305206825 00000100:00000001:1.0:1788879595.002142:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305206284 00000100:00000001:1.0:1788879595.002145:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305206825 00000100:00000001:1.0:1788879595.002147:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879595.002149:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.002150:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879595.002154:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879595/real 1788879595] req@ffff94e5045e5880 x1875765290179072/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879595.002162:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879595.002163:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879595.002166:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879595.002167:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879595.002169:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002171:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879595.002172:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879595.002176:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290179072/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879595.002183:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002185:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879595.002187:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002189:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879595.002190:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002192:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879595.002207:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879595.002211:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879595.002214:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879595.002216:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879595.002219:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290179072:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879595.002223:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002226:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879595.002227:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879595.002230:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879595.002234:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879595.002237:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879595.002239:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879595.002245:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879595.002247:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879595.002250:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879595.002252:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002253:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879595.002254:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879595.002256:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879595.002259:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879595.002261:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879595.002263:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879595.002266:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879595.002268:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879595.002270:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879595.002272:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002276:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002287:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879595.002290:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.002294:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879595.002297:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879595.002299:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879595.002302:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879595.002304:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879595.002308:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879595.002311:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879595.002314:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879595.002320:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290179200:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879595.002326:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879595.002330:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879595.002332:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879595.002334:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879595.002338:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.002347:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879595.002365:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:1.0:1788879595.002368:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879595.002378:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879595.002381:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879595.002383:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879595.002385:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.002386:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879595.002388:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.002390:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305206282 00000100:00000001:1.0:1788879595.002393:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305206825 00000100:00000001:1.0:1788879595.002394:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305206282 00000100:00000001:1.0:1788879595.002396:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305206825 00000100:00000001:1.0:1788879595.002397:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879595.002399:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.002400:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879595.002402:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879595/real 1788879595] req@ffff94e5045e6a00 x1875765290179200/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879595.002407:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879595.002408:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879595.002409:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879595.002411:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879595.002412:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002413:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879595.002414:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879595.002417:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290179200/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879595.002426:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002428:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879595.002430:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002431:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879595.002432:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002433:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879595.002442:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879595.002444:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879595.002446:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879595.002447:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879595.002449:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290179200:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879595.002452:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879595.002453:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879595.002454:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879595.002456:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879595.002457:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879595.002459:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879595.002460:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290179328:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879595.002462:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879595.002464:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879595.002465:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879595.002466:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879595.002468:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.002472:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879595.002473:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:1.0:1788879595.002475:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879595.002481:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879595.002484:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879595.002485:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879595.002486:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.002488:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879595.002489:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.002492:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305206282 00000100:00000001:1.0:1788879595.002494:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305206825 00000100:00000001:1.0:1788879595.002496:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305206282 00000100:00000001:1.0:1788879595.002497:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305206825 00000100:00000001:1.0:1788879595.002499:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879595.002500:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.002501:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879595.002503:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879595/real 1788879595] req@ffff94e5045e6680 x1875765290179328/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879595.002507:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879595.002508:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879595.002509:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879595.002510:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879595.002511:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002512:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879595.002513:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879595.002515:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290179328/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879595.002519:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002521:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879595.002523:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002524:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879595.002525:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002526:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879595.002535:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879595.002537:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879595.002538:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879595.002540:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879595.002542:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290179328:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879595.002545:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002546:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879595.002547:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879595.002548:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879595.002550:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879595.002550:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879595.002551:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879595.002554:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879595.002554:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879595.002556:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879595.002557:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002558:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879595.002559:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879595.002560:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879595.002561:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879595.002562:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879595.002562:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879595.002563:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879595.002564:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879595.002565:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879595.002565:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002566:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879595.002569:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879595.002569:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879595.002571:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879595.002572:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879595.065021:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:0.0:1788879595.065029:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879595.065032:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879595.065040:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:0.0:1788879595.065046:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879595.065051:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879595.065055:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879595.065060:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879595.065063:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879595.769040:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879595.769047:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879595.769052:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879595.769055:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879595.769057:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879595.769061:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879595.769063:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879595.769065:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879595.769067:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879595.769068:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879595.769089:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879595.769091:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879595.769093:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879596.089074:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879596.089084:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879596.089086:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879596.089093:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879596.089097:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879596.089101:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879596.089104:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879596.089107:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879596.089111:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879596.793071:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879596.793080:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879596.793088:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879596.793089:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879596.793115:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879596.793118:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879596.793122:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879597.113002:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879597.113010:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879597.113012:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879597.113018:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879597.113022:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879597.113027:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879597.113029:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879597.113032:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879597.113035:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879597.817096:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879597.817109:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879597.817113:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879597.817118:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879597.817126:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879597.817129:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879597.817132:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879597.817135:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879597.817140:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879597.817142:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879597.817228:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879597.817235:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879597.817241:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879598.137010:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879598.137019:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879598.137023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879598.137029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879598.137034:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879598.137039:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879598.137043:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879598.137046:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879598.137050:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879598.841092:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879598.841112:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879598.841119:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879598.841120:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879598.841166:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879598.841170:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879598.841173:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879599.161009:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879599.161016:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879599.161019:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879599.161025:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879599.161029:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879599.161034:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879599.161036:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879599.161039:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879599.161042:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879599.865118:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879599.865140:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879599.865147:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879599.865149:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879599.865225:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879599.865229:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879599.865233:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879600.121076:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879600.121090:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879600.121094:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879600.121100:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879600.121107:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879600.121118:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10538 00000100:00080000:0.0:1788879600.121134:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879600.121138:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879600.121141:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879600.121153:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879600.121157:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879600.121163:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879600.121168:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879600.121172:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879600.121176:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879600.121183:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879600.121188:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879600.121194:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879600.121199:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879600.121203:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879600.121207:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879600.121211:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879600.121232:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879600.121235:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879600.121242:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879600.121249:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879600.121270:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879600.121317:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879600.121339:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290179456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879600.121368:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879600.121375:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879600.121402:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879600.121408:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879600.121413:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879600.121416:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879600.121418:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879600.121423:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879600.121427:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10538 00000100:00080000:0.0:1788879600.121434:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879600.121437:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879600.121439:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879600.121446:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879600.121447:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1788879600.121449:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879600.121462:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879600.121464:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1788879600.121473:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879600.121474:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879600.121486:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879600.121486:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879600.121501:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879600.121502:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879600.121523:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1788879600.121525:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879600.121534:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879600.121536:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879600.121538:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879600.121549:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879600.121554:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788879600.121567:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290179456:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788879600.121568:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.121581:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879600.121587:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788879600.121591:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879600.121593:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879600.121600:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:0.0:1788879600.121606:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879600.121610:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788879600.121628:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879600.121636:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.121646:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1788879600.121647:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879600.121651:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 02000000:00000001:0.0:1788879600.121655:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.121659:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879600.121667:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879600.121673:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879600.121679:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879600.121690:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879600.121701:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290179584/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788879600.121720:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879600.121756:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879600.121760:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879600.121764:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879600.121765:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.121771:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.121777:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879600.121778:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.121786:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879600.121796:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879600.121796:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305211945 00000100:00080000:0.0:1788879600.121808:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879600.121810:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879600.121814:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879600.121816:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788879600.121822:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879600.121831:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305211945 00000100:00080000:0.0:1788879600.121832:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10538 00000100:00000001:1.0:1788879600.121853:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:0.0:1788879600.121854:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879600.121862:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879600.121863:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879600.121873:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879600.121874:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.121891:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:0.0:1788879600.121893:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879600.121897:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:1.0:1788879600.121970:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879600/real 1788879600] req@ffff94e508979c00 x1875765290179456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788879600.121985:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000400:00000001:1.0:1788879600.121989:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:0.0:1788879600.122002:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000400:00000001:1.0:1788879600.122003:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879600.122014:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879600.122017:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879600.122019:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879600.122029:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879600.122036:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879600.122037:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879600.122051:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879600.122052:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.122060:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879600.122062:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879600.122072:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788879600.122076:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290179456/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788879600.122079:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879600.122095:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879600.122105:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879600.122119:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788879600.122121:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879600.122129:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.122130:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879600.122144:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879600.122147:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879600.122149:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1788879600.122152:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.122154:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879600.122165:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879600.122167:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:0.0:1788879600.122174:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879600.122176:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1788879600.122180:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879600.122183:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1788879600.122192:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879600.122192:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879600.122202:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1788879600.122212:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290179712/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879600.122215:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290179456:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879600.122238:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.122247:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879600.122253:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879600.122260:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879600.122266:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1788879600.122272:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000020:00000001:1.0:1788879600.122273:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879600.122283:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:0.0:1788879600.122288:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879600.122292:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879600.122300:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1788879600.122303:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.122306:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879600.122310:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.122315:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1788879600.122317:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879600.122327:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1788879600.122328:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879600.122336:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879600.122338:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879600.122346:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879600.122348:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879600.122358:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879600.122362:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879600.122362:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879600.122369:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1788879600.122380:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10538 00000100:00000001:1.0:1788879600.122380:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879600.122397:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1788879600.122401:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788879600.122402:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879600.122418:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879600.122420:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879600.122428:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:1.0:1788879600.122437:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290179584:192.168.203.129@tcp:38:kworker.0 00000020:00000001:0.0:1788879600.122440:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879600.122450:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879600.122451:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1788879600.122465:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788879600.122465:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879600.122474:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879600.122476:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788879600.122481:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879600.122484:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879600.122487:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879600.122489:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.122512:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1788879600.122514:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879600.122525:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:0.0:1788879600.122527:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879600.122540:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879600.122543:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1788879600.122547:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879600.122554:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879600.122556:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879600.122560:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879600.122569:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879600.122571:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879600.122577:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.122581:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879600.122588:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879600.122591:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.122595:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:0.0:1788879600.122598:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879600.122600:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879600.122606:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.122609:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305211404 00000100:00000001:0.0:1788879600.122618:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879600.122619:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305211945 02000000:00000001:0.0:1788879600.122630:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879600.122630:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305211404 00000100:00000001:1.0:1788879600.122643:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305211945 02000000:00000001:0.0:1788879600.122645:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.122647:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879600.122658:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.122662:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879600.122667:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879600/real 1788879600] req@ffff94e50897a300 x1875765290179584/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879600.122671:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879600.122690:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879600.122693:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:0.0:1788879600.122696:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290179840/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879600.122697:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879600.122701:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879600.122703:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.122715:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879600.122719:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879600.122760:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290179584/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879600.122772:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879600.122777:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879600.122784:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.122785:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879600.122805:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.122809:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879600.122810:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.122813:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.122825:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879600.122856:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879600.122861:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879600.122865:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879600.122874:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879600.122880:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290179584:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879600.122885:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879600.122887:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879600.122889:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879600.122892:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879600.122895:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879600.122897:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879600.122927:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290179712:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879600.122932:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879600.122934:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879600.122936:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879600.122938:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879600.122942:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.122949:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879600.122950:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:1.0:1788879600.122954:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879600.122967:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879600.122973:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879600.122975:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879600.122977:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.122979:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879600.122982:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.122986:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305211403 00000100:00000001:1.0:1788879600.122989:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305211945 00000100:00000001:1.0:1788879600.122992:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305211403 00000100:00000001:1.0:1788879600.122994:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305211945 00000100:00000001:1.0:1788879600.122997:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879600.122999:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.123001:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879600.123006:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879600/real 1788879600] req@ffff94e508978e00 x1875765290179712/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879600.123015:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879600.123017:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879600.123019:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879600.123021:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879600.123023:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.123024:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879600.123026:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879600.123030:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290179712/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879600.123037:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.123040:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879600.123042:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.123044:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879600.123045:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.123047:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879600.123064:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879600.123066:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879600.123069:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879600.123071:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879600.123074:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290179712:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879600.123078:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.123080:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879600.123082:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879600.123083:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879600.123086:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879600.123087:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879600.123089:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879600.123092:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879600.123094:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879600.123097:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879600.123099:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.123100:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879600.123102:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879600.123103:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879600.123105:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879600.123106:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879600.123107:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879600.123109:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879600.123110:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879600.123112:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879600.123114:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.123115:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.123119:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879600.123120:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.123123:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879600.123124:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879600.123126:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879600.123127:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879600.123129:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879600.123132:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879600.123134:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879600.123136:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879600.123139:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290179840:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879600.123143:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879600.123144:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879600.123146:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879600.123148:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879600.123151:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.123155:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879600.123157:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:1.0:1788879600.123161:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879600.123171:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879600.123175:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879600.123178:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879600.123179:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.123181:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879600.123184:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.123188:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305211403 00000100:00000001:1.0:1788879600.123191:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305211945 00000100:00000001:1.0:1788879600.123193:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305211403 00000100:00000001:1.0:1788879600.123195:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305211945 00000100:00000001:1.0:1788879600.123198:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879600.123200:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.123202:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879600.123206:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879600/real 1788879600] req@ffff94e50897ad80 x1875765290179840/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879600.123213:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879600.123215:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879600.123217:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879600.123218:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879600.123220:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.123221:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879600.123223:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879600.123226:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290179840/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879600.123234:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.123236:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879600.123239:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.123241:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879600.123242:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.123244:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879600.123260:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879600.123263:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879600.123266:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879600.123268:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879600.123271:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290179840:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879600.123278:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.123280:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879600.123282:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879600.123283:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879600.123285:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879600.123287:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879600.123288:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879600.123290:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879600.123292:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879600.123295:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879600.123296:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.123298:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.123301:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879600.123302:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.123304:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879600.123306:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879600.184982:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879600.184988:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879600.184991:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879600.185003:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879600.185007:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879600.185010:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.185014:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879600.185017:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879600.185019:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879600.889046:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879600.889071:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879600.889080:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879600.889082:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879600.889139:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879600.889143:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879600.889148:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879601.209071:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879601.209080:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879601.209087:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879601.209093:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879601.209097:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879601.209102:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879601.209105:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879601.209107:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879601.209110:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879601.913065:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879601.913073:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879601.913080:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879601.913081:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879601.913144:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879601.913147:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879601.913151:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879602.233040:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879602.233053:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879602.233057:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879602.233066:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879602.233074:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879602.233081:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879602.233087:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879602.233091:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879602.233096:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879602.937047:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879602.937055:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879602.937063:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879602.937066:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879602.937138:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879602.937145:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879602.937150:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879603.257015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879603.257024:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879603.257027:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879603.257034:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879603.257039:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879603.257044:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879603.257047:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879603.257051:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879603.257054:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879603.961062:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879603.961072:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879603.961079:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879603.961081:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879603.961141:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879603.961145:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879603.961150:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879604.280961:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879604.280981:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879604.280983:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879604.280989:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879604.280993:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879604.280996:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879604.280999:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879604.281002:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879604.281005:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879604.984994:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879604.985000:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879604.985004:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879604.985006:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879604.985048:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879604.985051:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879604.985054:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879605.241014:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879605.241024:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879605.241028:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879605.241032:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879605.241036:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879605.241043:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10544 00000100:00080000:2.0:1788879605.241072:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879605.241076:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879605.241078:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879605.241086:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879605.241089:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879605.241093:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879605.241096:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879605.241100:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879605.241101:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879605.241106:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879605.241110:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879605.241114:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879605.241117:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879605.241120:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879605.241123:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879605.241126:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879605.241136:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879605.241138:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879605.241141:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879605.241144:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879605.241149:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879605.241160:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879605.241170:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290179968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879605.241182:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879605.241186:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879605.241198:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879605.241202:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879605.241206:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879605.241208:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879605.241210:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879605.241214:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879605.241217:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10544 00000100:00000001:1.0:1788879605.241218:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879605.241223:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788879605.241224:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879605.241227:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879605.241229:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879605.241231:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879605.241232:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879605.241234:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879605.241237:0:178195:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1788879605.241239:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1788879605.241240:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1788879605.241242:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1788879605.241244:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:1.0:1788879605.241246:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879605.241250:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00100000:1.0:1788879605.241253:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290179968:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879605.241255:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879605.241258:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879605.241260:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879605.241264:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1788879605.241265:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788879605.241267:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879605.241269:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879605.241270:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879605.241273:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879605.241276:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879605.241279:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879605.241283:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879605.241285:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879605.241289:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879605.241292:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000008:00000001:2.0:1788879605.241293:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241295:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:2.0:1788879605.241298:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241300:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1788879605.241306:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879605.241309:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879605.241311:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879605.241313:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879605.241315:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879605.241321:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879605.241327:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290180096/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788879605.241337:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879605.241339:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879605.241343:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879605.241347:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241350:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788879605.241351:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879605.241356:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788879605.241357:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879605.241360:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879605.241361:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879605.241364:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:2.0:1788879605.241365:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879605.241369:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879605.241370:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879605.241373:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305217065 00000100:00080000:2.0:1788879605.241376:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10544 00000100:00080000:2.0:1788879605.241381:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879605.241384:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788879605.241384:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879605.241389:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879605.241392:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879605.241393:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879605.241395:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879605.241398:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305217065 00000020:00000001:2.0:1788879605.241398:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879605.241402:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879605.241404:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879605.241406:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879605.241408:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788879605.241412:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879605.241413:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241418:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879605.241419:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879605.241422:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1788879605.241425:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879605/real 1788879605] req@ffff94e5045e4700 x1875765290179968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879605.241427:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879605.241433:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879605.241439:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879605.241443:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000400:00000001:1.0:1788879605.241445:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:2.0:1788879605.241447:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879605.241448:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879605.241452:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241453:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879605.241459:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:2.0:1788879605.241459:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879605.241463:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241464:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879605.241467:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879605.241470:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1788879605.241470:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879605.241475:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241476:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879605.241483:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879605.241485:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290179968/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879605.241491:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290180224/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879605.241509:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879605.241513:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879605.241516:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879605.241517:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241520:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879605.241522:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241524:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879605.241527:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879605.241527:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879605.241529:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879605.241533:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879605.241534:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879605.241536:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788879605.241538:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879605.241541:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1788879605.241542:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879605.241547:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788879605.241549:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00100000:1.0:1788879605.241553:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290179968:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1788879605.241557:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10544 00000100:00000001:1.0:1788879605.241559:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879605.241561:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879605.241562:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879605.241565:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788879605.241567:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879605.241568:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788879605.241570:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879605.241572:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879605.241574:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879605.241575:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879605.241577:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879605.241578:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000020:00000001:2.0:1788879605.241581:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788879605.241582:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:2.0:1788879605.241585:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788879605.241586:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879605.241590:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879605.241591:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879605.241592:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.241595:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879605.241595:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879605.241602:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879605.241603:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879605.241606:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241607:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241611:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879605.241611:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879605.241615:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879605.241618:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788879605.241618:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879605.241621:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879605.241625:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879605.241625:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879605.241631:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879605.241632:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879605.241633:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:2.0:1788879605.241635:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879605.241637:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788879605.241639:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879605.241643:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879605.241645:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290180096:192.168.203.129@tcp:38:kworker.0 02000000:00000001:2.0:1788879605.241648:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879605.241651:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241652:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788879605.241654:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879605.241656:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879605.241657:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879605.241661:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879605.241662:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879605.241667:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879605.241669:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879605.241672:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241682:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879605.241684:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00100000:2.0:1788879605.241686:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290180352/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879605.241689:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879605.241717:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000800:00000100:1.0:1788879605.241720:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879605.241721:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241726:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879605.241727:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241730:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879605.241733:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241734:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879605.241738:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241742:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305216523 00000100:00000001:1.0:1788879605.241744:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305217065 00000100:00000001:1.0:1788879605.241746:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305216523 00000100:00000001:1.0:1788879605.241748:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305217065 00000100:00000001:1.0:1788879605.241749:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879605.241751:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241753:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879605.241756:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879605/real 1788879605] req@ffff94e5045e7480 x1875765290180096/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879605.241762:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879605.241763:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879605.241765:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879605.241766:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879605.241767:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.241768:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879605.241769:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879605.241772:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290180096/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879605.241776:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.241778:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879605.241779:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.241780:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879605.241781:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.241782:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879605.241796:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879605.241798:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879605.241800:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879605.241802:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879605.241804:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290180096:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879605.241806:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879605.241807:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879605.241808:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879605.241809:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879605.241810:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879605.241811:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879605.241814:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290180224:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879605.241816:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879605.241817:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879605.241818:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879605.241819:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879605.241821:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241825:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879605.241826:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074559872 : -117763634991744 : ffff94e5045e6d80) 00000100:00000001:1.0:1788879605.241828:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879605.241834:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879605.241837:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879605.241838:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879605.241839:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241840:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879605.241842:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241844:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305216521 00000100:00000001:1.0:1788879605.241845:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305217065 00000100:00000001:1.0:1788879605.241847:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305216521 00000100:00000001:1.0:1788879605.241848:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305217065 00000100:00000001:1.0:1788879605.241850:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879605.241851:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241853:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879605.241855:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879605/real 1788879605] req@ffff94e5045e6d80 x1875765290180224/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879605.241860:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879605.241861:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879605.241862:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879605.241863:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879605.241864:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.241866:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879605.241866:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879605.241869:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6d80 x1875765290180224/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879605.241873:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.241874:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879605.241875:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.241876:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879605.241877:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.241878:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879605.241890:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879605.241891:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879605.241893:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879605.241894:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879605.241896:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290180224:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879605.241898:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.241899:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879605.241900:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879605.241922:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879605.241924:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879605.241925:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879605.241926:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879605.241927:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879605.241928:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879605.241930:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879605.241931:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.241932:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879605.241933:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879605.241934:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879605.241935:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879605.241936:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879605.241937:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879605.241938:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879605.241939:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879605.241940:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879605.241941:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.241942:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.241944:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879605.241945:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241947:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879605.241948:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879605.241948:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879605.241949:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879605.241950:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879605.241952:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879605.241953:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879605.241954:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879605.241956:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290180352:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879605.241959:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879605.241960:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879605.241961:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879605.241962:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879605.241964:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241967:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879605.241968:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:1.0:1788879605.241971:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879605.241978:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879605.241981:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879605.241982:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879605.241983:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241984:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879605.241986:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241988:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305216522 00000100:00000001:1.0:1788879605.241990:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305217065 00000100:00000001:1.0:1788879605.241992:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305216522 00000100:00000001:1.0:1788879605.241993:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305217065 00000100:00000001:1.0:1788879605.241995:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879605.241996:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.241997:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879605.242000:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879605/real 1788879605] req@ffff94e5045e4e00 x1875765290180352/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879605.242004:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879605.242005:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879605.242006:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879605.242007:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879605.242008:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.242010:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879605.242011:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879605.242013:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290180352/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879605.242017:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.242019:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879605.242020:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.242021:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879605.242021:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.242023:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879605.242034:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879605.242036:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879605.242038:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879605.242040:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879605.242042:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290180352:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879605.242044:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.242046:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879605.242046:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879605.242047:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879605.242049:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879605.242050:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879605.242051:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879605.242052:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879605.242053:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879605.242054:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879605.242055:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.242056:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.242057:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879605.242059:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.242060:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879605.242061:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879605.304979:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879605.304987:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879605.304988:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879605.304995:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879605.304999:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879605.305003:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.305007:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879605.305010:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879605.305013:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879606.009067:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879606.009076:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879606.009085:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879606.009089:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879606.009092:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879606.009097:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879606.009102:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879606.009105:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879606.009108:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879606.009111:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879606.009141:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879606.009146:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879606.009149:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879606.329036:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879606.329047:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879606.329050:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879606.329057:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879606.329062:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879606.329067:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879606.329071:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879606.329075:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879606.329078:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879607.033086:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879607.033099:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879607.033103:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879607.033108:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879607.033115:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879607.033118:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879607.033121:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879607.033124:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879607.033128:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879607.033130:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879607.033195:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879607.033201:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879607.033205:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879607.352978:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879607.352987:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879607.352989:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879607.352994:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879607.352997:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879607.353000:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879607.353003:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879607.353006:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879607.353008:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879608.057088:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879608.057097:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879608.057106:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879608.057107:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879608.057146:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879608.057149:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879608.057153:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879608.376962:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879608.376970:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879608.376972:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879608.376977:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879608.376980:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879608.376995:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879608.376998:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879608.377001:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879608.377003:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879609.081030:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879609.081037:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879609.081043:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879609.081044:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879609.081076:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879609.081079:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879609.081082:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879609.400966:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879609.400972:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879609.400974:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879609.400978:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879609.400981:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879609.400984:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879609.400986:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879609.400989:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879609.400991:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879610.104971:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879610.104976:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879610.104980:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879610.104980:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879610.104997:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879610.104999:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879610.105001:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879610.360967:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879610.360976:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879610.360979:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879610.360983:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879610.360988:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879610.360995:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10549 00000100:00080000:2.0:1788879610.361006:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879610.361009:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879610.361012:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879610.361020:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879610.361023:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879610.361026:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879610.361029:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879610.361031:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879610.361033:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879610.361038:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879610.361040:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879610.361044:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879610.361047:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879610.361051:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879610.361054:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879610.361064:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879610.361074:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879610.361076:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879610.361080:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879610.361083:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879610.361088:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879610.361100:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879610.361109:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4000 x1875765290180480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879610.361121:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879610.361125:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879610.361137:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879610.361144:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879610.361147:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879610.361149:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879610.361151:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879610.361154:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879610.361157:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10549 00000100:00080000:2.0:1788879610.361162:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879610.361164:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879610.361166:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879610.361170:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879610.361171:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788879610.361172:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879610.361176:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879610.361177:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788879610.361180:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879610.361181:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879610.361185:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879610.361185:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879610.361188:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879610.361189:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879610.361193:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1788879610.361195:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788879610.361198:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879610.361199:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879610.361202:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879610.361203:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1788879610.361209:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290180480:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879610.361210:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879610.361217:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879610.361217:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879610.361222:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788879610.361222:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879610.361226:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879610.361229:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:2.0:1788879610.361229:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879610.361234:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879610.361235:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879610.361238:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879610.361244:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879610.361246:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879610.361248:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879610.361250:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879610.361251:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361252:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879610.361255:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074548224 : -117763635003392 : ffff94e5045e4000) 00000100:00000001:2.0:1788879610.361258:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361259:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1788879610.361268:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290180608/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879610.361278:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879610.361280:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879610.361284:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879610.361287:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879610.361290:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879610.361292:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879610.361294:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1788879610.361295:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879610.361298:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879610.361303:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10549 00000100:00000001:1.0:1788879610.361307:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:2.0:1788879610.361308:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879610.361312:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879610.361313:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879610.361314:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879610.361317:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879610.361319:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879610.361320:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:2.0:1788879610.361322:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879610.361326:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879610.361326:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879610.361331:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879610.361332:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305222185 00000100:00000001:2.0:1788879610.361335:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879610.361338:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879610.361342:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879610.361343:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788879610.361346:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879610.361349:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879610.361350:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305222185 02000000:00000001:2.0:1788879610.361352:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879610.361356:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1788879610.361356:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361361:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879610.361361:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879610.361365:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879610.361366:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361370:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000008:00000001:2.0:1788879610.361372:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879610.361375:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879610.361377:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879610/real 1788879610] req@ffff94e5045e4000 x1875765290180480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879610.361381:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879610.361385:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879610.361389:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:1.0:1788879610.361391:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1788879610.361391:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879610.361393:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879610.361394:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879610.361399:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879610.361401:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361402:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879610.361404:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.361407:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879610.361410:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879610.361411:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290180736/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879610.361416:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4000 x1875765290180480/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879610.361436:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879610.361436:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879610.361441:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879610.361441:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361446:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879610.361447:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361448:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879610.361451:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879610.361452:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879610.361454:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879610.361456:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879610.361459:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00080000:1.0:1788879610.361461:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788879610.361464:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879610.361467:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00080000:2.0:1788879610.361468:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879610.361472:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788879610.361474:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00100000:1.0:1788879610.361478:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290180480:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1788879610.361479:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10549 00000100:00000001:1.0:1788879610.361485:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879610.361485:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879610.361488:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879610.361490:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788879610.361490:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879610.361494:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788879610.361494:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879610.361498:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879610.361500:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879610.361501:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879610.361502:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879610.361505:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000020:00000001:2.0:1788879610.361506:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788879610.361508:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:2.0:1788879610.361510:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788879610.361511:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879610.361515:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879610.361516:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879610.361520:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879610.361520:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879610.361524:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879610.361527:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879610.361530:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879610.361531:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361534:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879610.361535:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879610.361537:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879610.361540:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788879610.361540:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879610.361543:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788879610.361545:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361546:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879610.361548:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879610.361550:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879610.361551:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879610.361554:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:2.0:1788879610.361555:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879610.361556:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788879610.361561:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879610.361562:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290180608:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879610.361564:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361566:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879610.361569:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879610.361570:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879610.361570:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879610.361571:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879610.361574:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879610.361575:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879610.361577:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879610.361580:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879610.361583:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788879610.361583:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361586:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00000001:2.0:1788879610.361589:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361590:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1788879610.361597:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290180864/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788879610.361604:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879610.361609:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879610.361610:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879610.361612:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879610.361614:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361616:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361619:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879610.361620:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361622:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361626:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305221643 00000100:00000001:1.0:1788879610.361628:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305222185 00000100:00000001:1.0:1788879610.361631:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305221643 00000100:00000001:1.0:1788879610.361633:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305222185 00000100:00000001:1.0:1788879610.361635:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879610.361636:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361638:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879610.361642:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879610/real 1788879610] req@ffff94e5045e7b80 x1875765290180608/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879610.361648:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879610.361650:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879610.361651:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879610.361653:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879610.361654:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.361655:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879610.361657:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879610.361672:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290180608/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879610.361678:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.361680:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879610.361682:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.361683:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879610.361683:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.361684:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879610.361700:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879610.361702:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879610.361704:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879610.361706:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879610.361710:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290180608:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879610.361715:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879610.361717:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879610.361718:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879610.361721:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879610.361723:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879610.361725:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879610.361728:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290180736:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879610.361732:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879610.361733:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879610.361735:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879610.361737:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879610.361740:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361747:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879610.361749:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:1.0:1788879610.361753:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879610.361766:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879610.361771:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879610.361774:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879610.361775:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361777:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879610.361780:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361784:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305221641 00000100:00000001:1.0:1788879610.361787:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305222185 00000100:00000001:1.0:1788879610.361790:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305221641 00000100:00000001:1.0:1788879610.361792:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305222185 00000100:00000001:1.0:1788879610.361795:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879610.361797:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361799:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879610.361804:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879610/real 1788879610] req@ffff94e5045e4e00 x1875765290180736/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879610.361819:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879610.361820:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879610.361822:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879610.361823:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879610.361824:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.361825:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879610.361827:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879610.361830:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290180736/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879610.361835:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.361837:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879610.361838:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.361840:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879610.361840:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.361841:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879610.361855:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879610.361856:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879610.361858:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879610.361860:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879610.361862:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290180736:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879610.361864:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.361866:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879610.361867:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879610.361868:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879610.361870:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879610.361871:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879610.361872:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879610.361873:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879610.361874:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879610.361876:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879610.361877:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.361879:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879610.361880:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879610.361881:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879610.361882:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879610.361883:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879610.361884:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879610.361885:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879610.361886:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879610.361887:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879610.361888:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.361889:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.361891:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879610.361892:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361893:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879610.361894:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879610.361894:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879610.361895:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879610.361896:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879610.361897:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879610.361899:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879610.361900:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879610.361928:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290180864:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879610.361932:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879610.361935:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879610.361937:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879610.361939:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879610.361942:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361948:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879610.361950:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074559872 : -117763634991744 : ffff94e5045e6d80) 00000100:00000001:1.0:1788879610.361954:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879610.361965:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879610.361968:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879610.361970:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879610.361971:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361972:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879610.361974:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361976:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305221642 00000100:00000001:1.0:1788879610.361978:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305222185 00000100:00000001:1.0:1788879610.361980:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305221642 00000100:00000001:1.0:1788879610.361982:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305222185 00000100:00000001:1.0:1788879610.361984:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879610.361985:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.361987:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879610.361989:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879610/real 1788879610] req@ffff94e5045e6d80 x1875765290180864/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879610.361994:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879610.361995:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879610.361996:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879610.361998:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879610.361999:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.362000:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879610.362001:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879610.362004:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6d80 x1875765290180864/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879610.362009:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.362010:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879610.362012:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.362013:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879610.362013:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.362014:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879610.362028:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879610.362030:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879610.362031:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879610.362033:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879610.362034:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290180864:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879610.362037:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.362039:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879610.362040:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879610.362041:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879610.362043:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879610.362044:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879610.362045:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879610.362046:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879610.362047:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879610.362048:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879610.362049:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.362051:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.362053:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879610.362054:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.362063:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879610.362064:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879610.424957:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879610.424960:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879610.424962:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879610.424965:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879610.424967:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879610.424970:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.424971:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879610.424973:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879610.424975:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879611.128994:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879611.129000:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879611.129005:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879611.129006:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879611.129067:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879611.129071:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879611.129075:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879611.448985:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879611.448992:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879611.448994:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879611.448999:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879611.449004:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879611.449008:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879611.449011:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879611.449014:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879611.449017:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879612.153044:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879612.153053:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879612.153060:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879612.153062:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879612.153125:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879612.153128:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879612.153132:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879612.473052:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879612.473065:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879612.473067:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879612.473076:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879612.473081:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879612.473087:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879612.473091:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879612.473095:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879612.473099:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879613.177040:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879613.177048:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879613.177054:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879613.177056:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879613.177085:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879613.177088:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879613.177092:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879613.497045:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879613.497057:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879613.497059:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879613.497068:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879613.497074:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879613.497079:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879613.497082:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879613.497086:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879613.497090:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879614.201141:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879614.201151:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879614.201158:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879614.201160:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879614.201215:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879614.201218:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879614.201222:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879614.521024:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879614.521033:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879614.521036:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879614.521042:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879614.521047:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879614.521052:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879614.521056:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879614.521060:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879614.521064:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879615.225030:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879615.225037:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879615.225042:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879615.225045:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879615.225047:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879615.225050:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879615.225053:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879615.225054:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879615.225057:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879615.225060:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879615.225167:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879615.225171:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879615.225174:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879615.481128:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879615.481141:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879615.481145:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879615.481150:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879615.481157:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879615.481166:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10554 00000100:00080000:3.0:1788879615.481180:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879615.481184:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879615.481187:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879615.481197:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879615.481200:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879615.481205:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879615.481208:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879615.481211:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879615.481213:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879615.481219:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879615.481222:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879615.481226:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879615.481231:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879615.481235:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.481237:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879615.481240:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879615.481253:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879615.481255:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.481259:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879615.481263:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879615.481269:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.481284:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879615.481297:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290180992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879615.481311:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879615.481317:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.481341:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879615.481346:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879615.481350:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879615.481352:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879615.481354:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879615.481358:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879615.481361:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10554 00000100:00080000:3.0:1788879615.481364:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879615.481367:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879615.481368:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879615.481372:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879615.481374:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879615.481378:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879615.481381:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879615.481384:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879615.481386:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879615.481391:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879615.481395:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879615.481398:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879615.481405:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879615.481409:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.481411:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879615.481416:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879615.481419:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:3.0:1788879615.481421:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.481428:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.481430:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879615.481435:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788879615.481435:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879615.481447:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.481448:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879615.481458:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879615.481460:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879615.481474:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.481480:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.481483:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879615.481494:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1788879615.481498:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290181120/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879615.481501:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879615.481520:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.481536:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879615.481539:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.481547:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879615.481554:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788879615.481558:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290180992:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1788879615.481560:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879615.481564:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879615.481569:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879615.481578:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879615.481586:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788879615.481589:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10554 00000100:00080000:3.0:1788879615.481599:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788879615.481600:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788879615.481605:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879615.481608:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.481614:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879615.481619:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1788879615.481647:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879615.481653:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1788879615.481656:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879615.481659:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879615.481679:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879615.481684:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879615.481687:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879615.481697:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879615.481701:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879615.481704:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.481705:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 02000000:00000001:3.0:1788879615.481720:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879615.481721:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788879615.481734:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879615.481741:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.481756:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879615.481760:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879615.481764:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.481767:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879615.481775:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879615.481777:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.481779:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879615.481781:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879615.481785:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.481794:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879615.481808:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290181248/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879615.481824:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879615.481826:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.481834:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.481838:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305227305 00000100:00000001:3.0:1788879615.481846:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879615.481856:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879615.481864:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788879615.481865:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879615.481874:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879615.481878:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879615.481887:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879615.481888:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305227305 00000100:00000001:1.0:1788879615.481899:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879615.481981:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879615.481982:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10554 00000100:00000001:1.0:1788879615.481993:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879615.481999:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879615.482000:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.482010:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879615.482011:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879615.482019:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879615.482025:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:3.0:1788879615.482029:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879615.482030:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000020:00000001:3.0:1788879615.482035:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879615.482037:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788879615.482041:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879615.482042:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879615.482048:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788879615.482050:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879615.482052:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879615.482057:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.482062:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879615.482065:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879615.482068:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879615.482070:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879615.482077:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879615.482079:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1788879615.482089:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290181120:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879615.482091:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.482101:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879615.482101:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879615.482112:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1788879615.482114:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879615.482116:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879615.482126:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879615.482127:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879615.482138:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879615.482139:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788879615.482141:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879615.482148:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.482151:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.482153:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879615.482157:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 02000000:00000001:3.0:1788879615.482159:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879615.482164:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1788879615.482164:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.482176:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879615.482180:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879615.482184:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.482196:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.482197:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.482209:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305226764 00000100:00100000:3.0:1788879615.482217:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290181376/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879615.482219:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305227305 00000100:00000001:1.0:1788879615.482237:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305226764 00000100:00000001:1.0:1788879615.482250:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305227305 00000100:00000001:1.0:1788879615.482268:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879615.482268:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879615.482278:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879615.482278:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.482301:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879615.482306:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.482309:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879615.482314:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879615.482320:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879615.482325:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879615.482329:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879615.482337:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290181248:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879615.482343:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879615.482347:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879615.482349:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879615.482351:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879615.482357:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.482365:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879615.482366:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:1.0:1788879615.482370:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879615.482381:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.482384:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305226762 00000100:00000001:1.0:1788879615.482386:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305227305 00000100:00000001:1.0:1788879615.482388:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305226762 00000100:00000001:1.0:1788879615.482389:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305227305 00000100:00000001:1.0:1788879615.482391:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879615.482393:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.482395:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.482396:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879615.482399:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879615.482400:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788879615.482402:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879615.482404:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879615.482405:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879615.482407:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879615.482408:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879615.482410:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879615.482412:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879615.482413:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879615.482417:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290181376:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879615.482420:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879615.482422:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879615.482424:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879615.482425:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879615.482428:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.482432:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879615.482433:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:1.0:1788879615.482436:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879615.482442:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.482445:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305226762 00000100:00000001:1.0:1788879615.482447:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305227305 00000100:00000001:1.0:1788879615.482448:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305226762 00000100:00000001:1.0:1788879615.482450:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305227305 00000100:00000001:1.0:1788879615.482452:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879615.482453:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.482455:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.482456:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879615.482458:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879615.482459:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788879615.482460:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879615.482461:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879615.482462:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.482463:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879615.545101:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879615.545115:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879615.545117:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879615.545125:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879615.545129:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879615.545134:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879615.545137:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879615.545141:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879615.545144:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879616.249027:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879616.249035:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879616.249043:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879616.249044:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879616.249070:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879616.249073:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879616.249077:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879616.569044:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879616.569057:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879616.569061:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879616.569070:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879616.569077:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879616.569084:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879616.569089:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879616.569094:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879616.569099:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879617.273063:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879617.273072:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879617.273075:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879617.273079:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879617.273085:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879617.273087:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879617.273090:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879617.273092:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879617.273097:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879617.273098:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879617.273131:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879617.273134:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879617.273137:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879617.593016:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879617.593025:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879617.593028:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879617.593035:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879617.593040:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879617.593045:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879617.593049:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879617.593052:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879617.593056:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879618.297041:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879618.297049:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879618.297056:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879618.297058:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879618.297095:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879618.297099:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879618.297104:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788879618.553213:0:2350:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1788879618.553266:0:2350:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788879618.553289:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879618.553308:0:2350:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879618.553329:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879618.553335:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879618.553338:0:2350:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879618.553354:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879618.553364:0:2350:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879618.553365:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879618.553371:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879618.553371:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879618.553376:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879618.553377:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879618.553383:0:2350:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879618.553395:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:1.0:1788879618.553396:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879615/real 1788879618] req@ffff94e50449fb80 x1875765290181120/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879618.553410:0:2350:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879618.553419:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879618.553422:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879618.553425:0:2350:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879618.553438:0:2350:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000400:00000001:1.0:1788879618.553449:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879618.553454:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879618.553463:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879618.553467:0:2350:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879618.553470:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:0.0:1788879618.553476:0:2350:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879618.553476:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879618.553482:0:2350:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879618.553485:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879618.553488:0:2350:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879618.553491:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879618.553508:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290181120/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879618.553521:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.553526:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879618.553530:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.553533:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879618.553536:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.553539:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879618.553564:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879618.553570:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879618.553578:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879618.553582:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879618.553588:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290181120:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879618.553595:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879618.553625:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879615/real 1788879618] req@ffff94e50449d880 x1875765290181248/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879618.553639:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879618.553641:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879618.553646:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879618.553649:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879618.553651:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.553653:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879618.553656:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879618.553662:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290181248/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879618.553678:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.553681:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879618.553684:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.553686:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879618.553688:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.553691:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879618.553711:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879618.553714:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879618.553719:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879618.553723:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879618.553727:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290181248:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879618.553733:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879618.553739:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879615/real 1788879618] req@ffff94e50449c380 x1875765290180992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879618.553749:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879618.553751:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879618.553756:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879618.553758:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879618.553760:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.553762:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879618.553764:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879618.553768:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290180992/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879618.553777:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.553781:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879618.553783:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.553785:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879618.553787:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.553790:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879618.553792:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879618.553795:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879618.553800:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879618.553802:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879618.553807:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290180992:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879618.553813:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879618.553816:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879618.553818:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879618.553823:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879618.553830:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879618.553834:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879618.553837:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879618.553843:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879618.553853:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879618.553858:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879618.553860:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.553863:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879618.553865:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879618.553867:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879618.553870:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879618.553871:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879618.553874:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879618.553876:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879618.553878:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879618.553881:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879618.553882:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.553885:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879618.553886:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879618.553888:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879618.553891:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879618.553893:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879618.553895:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879618.553897:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879618.553900:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879618.553936:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879618.553939:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.553942:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879618.553945:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879618.553947:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879618.553950:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879618.553956:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879615/real 1788879618] req@ffff94e50449f800 x1875765290181376/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879618.553968:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879618.553970:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879618.553974:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879618.553977:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879618.553979:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.553982:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879618.553984:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879618.553989:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290181376/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879618.553998:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.554002:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879618.554005:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.554008:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879618.554009:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.554012:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879618.554033:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879618.554037:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879618.554043:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879618.554047:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879618.554052:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290181376:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879618.554057:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.554060:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879618.554062:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879618.554064:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879618.554067:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879618.554069:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879618.554071:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879618.554074:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879618.554076:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879618.554078:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879618.554080:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.554083:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.554093:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879618.554096:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879618.554100:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879618.554101:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879618.616980:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879618.616987:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879618.616989:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879618.616994:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879618.616997:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879618.617001:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.617004:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879618.617007:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879618.617010:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879619.321041:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879619.321049:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879619.321055:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879619.321057:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879619.321136:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879619.321142:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879619.321147:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879619.641036:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879619.641045:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879619.641048:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879619.641055:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879619.641060:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879619.641066:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879619.641069:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879619.641073:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879619.641077:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879620.345034:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879620.345043:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879620.345050:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879620.345052:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879620.345079:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879620.345083:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879620.345087:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879620.601086:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879620.601099:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879620.601104:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879620.601109:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879620.601116:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879620.601127:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10559 00000100:00080000:2.0:1788879620.601143:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879620.601149:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879620.601153:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879620.601167:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879620.601172:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879620.601178:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879620.601184:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879620.601188:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879620.601191:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879620.601200:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879620.601207:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879620.601212:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879620.601219:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879620.601225:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879620.601230:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879620.601234:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879620.601249:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879620.601253:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879620.601259:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879620.601263:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879620.601271:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879620.601290:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879620.601307:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290181504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879620.601329:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879620.601338:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879620.601363:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879620.601370:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879620.601377:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879620.601380:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879620.601384:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879620.601391:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879620.601397:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10559 00000100:00080000:2.0:1788879620.601406:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879620.601409:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879620.601412:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879620.601415:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788879620.601419:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879620.601423:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1788879620.601425:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879620.601432:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1788879620.601433:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879620.601440:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1788879620.601441:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879620.601448:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879620.601449:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879620.601456:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879620.601461:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1788879620.601464:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788879620.601469:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879620.601472:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879620.601477:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879620.601480:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1788879620.601491:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290181504:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879620.601496:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879620.601500:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879620.601505:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879620.601509:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788879620.601514:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879620.601516:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879620.601524:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:2.0:1788879620.601526:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879620.601534:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788879620.601536:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879620.601548:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879620.601559:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879620.601561:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879620.601565:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879620.601568:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879620.601570:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788879620.601574:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879620.601614:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.601615:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00000001:1.0:1788879620.601629:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1788879620.601632:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290181632/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879620.601653:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879620.601659:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879620.601667:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879620.601673:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879620.601681:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879620.601684:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879620.601687:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879620.601694:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000800:00000100:1.0:1788879620.601705:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788879620.601706:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10559 00000100:00080000:2.0:1788879620.601722:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879620.601729:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879620.601734:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879620.601734:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879620.601749:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1788879620.601750:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879620.601758:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879620.601762:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879620.601766:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879620.601772:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:2.0:1788879620.601775:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879620.601785:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879620.601789:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879620.601790:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879620.601802:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879620.601813:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.601815:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305232425 02000000:00000001:2.0:1788879620.601832:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879620.601837:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1788879620.601840:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879620.601851:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879620.601854:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879620.601858:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879620.601863:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879620.601866:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879620.601874:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879620.601877:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.601880:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305232425 00000100:00000001:2.0:1788879620.601883:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879620.601889:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879620.601897:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.601899:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879620.601954:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879620.601961:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.601966:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879620.601972:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879620.601983:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879620/real 1788879620] req@ffff94e5045e7480 x1875765290181504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879620.601995:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290181760/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1788879620.602040:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788879620.602041:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:1.0:1788879620.602050:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879620.602052:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879620.602062:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.602065:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:2.0:1788879620.602068:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879620.602075:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00080000:2.0:1788879620.602079:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879620.602084:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879620.602086:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879620.602092:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879620.602096:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879620.602101:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879620.602104:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879620.602114:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10559 00000100:00100000:1.0:1788879620.602126:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290181504/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1788879620.602128:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879620.602138:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879620.602143:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879620.602163:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.602170:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1788879620.602178:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879620.602180:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788879620.602186:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879620.602189:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1788879620.602194:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879620.602197:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788879620.602205:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879620.602209:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879620.602214:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879620.602218:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879620.602222:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879620.602235:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879620.602241:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879620.602251:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879620.602256:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 02000000:00000001:2.0:1788879620.602258:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879620.602270:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1788879620.602270:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879620.602282:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879620.602291:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290181504:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879620.602292:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879620.602299:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879620.602310:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.602313:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879620.602316:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.602325:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788879620.602331:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879620.602332:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879620.602337:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879620.602341:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788879620.602346:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879620.602352:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879620.602354:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:2.0:1788879620.602358:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879620.602362:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879620.602370:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:2.0:1788879620.602373:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879620.602379:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879620.602387:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00100000:2.0:1788879620.602391:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290181888/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879620.602396:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879620.602404:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.602416:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879620.602424:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879620.602432:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.602434:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879620.602440:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.602443:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.602454:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879620.602458:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879620.602462:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879620.602466:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879620.602469:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879620.602474:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879620.602480:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879620.602485:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879620.602495:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290181632:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879620.602503:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879620.602507:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879620.602511:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879620.602514:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879620.602520:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.602534:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879620.602537:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:1.0:1788879620.602545:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879620.602617:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879620.602629:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879620.602635:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879620.602638:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.602641:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879620.602647:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.602656:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305231884 00000100:00000001:1.0:1788879620.602663:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305232425 00000100:00000001:1.0:1788879620.602668:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305231884 00000100:00000001:1.0:1788879620.602672:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305232425 00000100:00000001:1.0:1788879620.602676:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879620.602680:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.602696:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879620.602705:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879620/real 1788879620] req@ffff94e5045e4700 x1875765290181632/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879620.602723:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879620.602725:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879620.602731:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879620.602735:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879620.602737:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.602741:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879620.602744:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879620.602751:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290181632/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879620.602772:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.602776:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879620.602781:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.602784:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879620.602786:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.602790:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879620.602815:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879620.602822:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879620.602829:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879620.602834:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879620.602840:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290181632:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879620.602848:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879620.602850:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879620.602853:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879620.602857:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879620.602861:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879620.602865:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879620.602871:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290181760:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879620.602884:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879620.602888:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879620.602891:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879620.602895:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879620.602943:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.602959:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879620.602962:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:1.0:1788879620.602969:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879620.602998:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879620.603008:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879620.603013:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879620.603016:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.603020:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879620.603026:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.603033:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305231883 00000100:00000001:1.0:1788879620.603039:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305232425 00000100:00000001:1.0:1788879620.603043:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305231883 00000100:00000001:1.0:1788879620.603047:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305232425 00000100:00000001:1.0:1788879620.603051:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879620.603055:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.603059:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879620.603069:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879620/real 1788879620] req@ffff94e5045e6680 x1875765290181760/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879620.603087:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879620.603090:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879620.603096:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879620.603100:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879620.603103:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.603107:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879620.603110:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879620.603119:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290181760/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879620.603147:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.603152:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879620.603157:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.603160:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879620.603163:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.603167:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879620.603192:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879620.603198:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879620.603205:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879620.603210:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879620.603217:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290181760:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879620.603225:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879620.603227:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879620.603230:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879620.603235:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879620.603239:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879620.603243:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879620.603250:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290181888:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879620.603256:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879620.603260:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879620.603263:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879620.603267:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879620.603274:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.603285:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879620.603288:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:1.0:1788879620.603296:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879620.603323:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879620.603333:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879620.603339:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879620.603342:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.603347:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879620.603353:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.603360:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305231883 00000100:00000001:1.0:1788879620.603365:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305232425 00000100:00000001:1.0:1788879620.603370:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305231883 00000100:00000001:1.0:1788879620.603374:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305232425 00000100:00000001:1.0:1788879620.603378:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879620.603383:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.603386:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879620.603396:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879620/real 1788879620] req@ffff94e5045e6a00 x1875765290181888/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879620.603415:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879620.603418:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879620.603423:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879620.603427:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879620.603430:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.603433:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879620.603436:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879620.603444:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290181888/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879620.603455:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.603468:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879620.603471:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.603473:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879620.603475:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.603477:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879620.603496:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879620.603499:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879620.603503:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879620.603507:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879620.603514:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290181888:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879620.603522:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.603527:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879620.603529:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879620.603533:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879620.603538:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879620.603541:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879620.603544:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879620.603549:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879620.603552:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879620.603556:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879620.603559:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.603563:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879620.603565:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879620.603568:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879620.603571:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879620.603573:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879620.603576:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879620.603579:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879620.603582:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879620.603585:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879620.603610:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.603614:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879620.603617:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879620.603619:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879620.603634:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879620.603637:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879620.603639:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879620.603642:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879620.603644:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879620.603647:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879620.603649:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.603654:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.603663:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879620.603667:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.603671:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879620.603674:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879620.665034:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879620.665042:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879620.665045:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879620.665053:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879620.665058:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879620.665064:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.665069:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879620.665074:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879620.665079:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879621.369047:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879621.369055:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879621.369062:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879621.369064:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879621.369091:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879621.369095:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879621.369099:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879621.689026:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879621.689035:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879621.689038:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879621.689044:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879621.689048:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879621.689053:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879621.689057:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879621.689061:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879621.689064:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879622.392984:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879622.392990:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879622.392995:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879622.392996:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879622.393026:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879622.393028:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879622.393031:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879622.712989:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879622.712996:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879622.712998:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879622.713002:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879622.713005:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879622.713009:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879622.713011:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879622.713013:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879622.713015:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879623.417007:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879623.417014:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879623.417020:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879623.417022:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879623.417090:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879623.417095:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879623.417099:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879623.736971:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879623.736979:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879623.736982:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879623.736987:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879623.736991:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879623.736995:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879623.736998:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879623.737002:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879623.737005:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879624.440973:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879624.440980:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879624.440985:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879624.440986:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879624.441004:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879624.441006:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879624.441008:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879624.760990:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879624.761001:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879624.761006:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879624.761030:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879624.761037:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879624.761044:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879624.761049:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879624.761054:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879624.761059:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879625.465054:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879625.465064:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879625.465075:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879625.465081:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879625.465085:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879625.465092:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879625.465097:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879625.465101:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879625.465105:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879625.465110:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879625.465141:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879625.465144:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879625.465148:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879625.721048:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879625.721062:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879625.721066:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879625.721072:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879625.721079:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879625.721088:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10564 00000100:00080000:0.0:1788879625.721102:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879625.721107:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879625.721110:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879625.721122:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879625.721126:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879625.721143:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879625.721147:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879625.721150:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879625.721153:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879625.721160:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879625.721164:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879625.721168:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879625.721173:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879625.721177:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.721180:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879625.721184:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879625.721196:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879625.721199:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.721203:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879625.721207:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879625.721214:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.721229:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879625.721243:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290182016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879625.721258:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879625.721264:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.721291:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879625.721297:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879625.721303:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879625.721305:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879625.721308:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879625.721312:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879625.721315:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10564 00000100:00080000:0.0:1788879625.721321:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879625.721324:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879625.721327:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879625.721332:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879625.721335:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879625.721338:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879625.721341:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879625.721344:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879625.721346:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879625.721351:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879625.721354:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879625.721357:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879625.721365:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879625.721368:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.721371:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879625.721376:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879625.721381:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.721384:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879625.721392:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879625.721394:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.721396:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879625.721399:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879625.721402:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.721410:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.721411:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788879625.721423:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978700 x1875765290182144/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879625.721424:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879625.721433:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879625.721442:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879625.721459:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.721464:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788879625.721471:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788879625.721474:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879625.721480:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879625.721487:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879625.721491:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879625.721497:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1788879625.721508:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290182016:192.168.203.129@tcp:250:kworker.0 00000100:00080000:0.0:1788879625.721510:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879625.721521:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879625.721524:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1788879625.721529:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1788879625.721532:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788879625.721537:0:178195:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:2.0:1788879625.721539:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879625.721546:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1788879625.721578:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10564 02000000:00000001:2.0:1788879625.721578:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879625.721592:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879625.721597:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879625.721600:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879625.721607:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879625.721608:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1788879625.721611:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879625.721614:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000020:00000001:0.0:1788879625.721617:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879625.721622:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1788879625.721624:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879625.721630:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879625.721633:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879625.721639:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879625.721643:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879625.721647:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879625.721652:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879625.721656:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.721659:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879625.721661:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879625.721665:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.721669:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879625.721676:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879625.721679:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.721682:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879625.721684:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879625.721688:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788879625.721694:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879625.721703:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.721713:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879625.721722:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879625.721727:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879625.721729:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897aa00 x1875765290182272/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879625.721731:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879625.721741:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.721747:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879625.721750:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305237545 00000100:00000001:0.0:1788879625.721755:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.721762:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879625.721766:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879625.721769:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1788879625.721771:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879625.721779:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879625.721782:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879625.721784:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305237545 00000100:00000001:0.0:1788879625.721787:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879625.721796:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10564 00000100:00000001:2.0:1788879625.721797:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:0.0:1788879625.721804:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879625.721806:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879625.721812:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879625.721815:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.721818:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879625.721825:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879625.721829:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879625.721831:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879625.721832:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:0.0:1788879625.721838:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879625.721844:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1788879625.721851:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879625/real 1788879625] req@ffff94e508978a80 x1875765290182016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879625.721858:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879625.721874:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000400:00000001:2.0:1788879625.721880:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788879625.721881:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879625.721886:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1788879625.721888:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879625.721896:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:0.0:1788879625.721897:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879625.721952:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879625.721958:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879625.721962:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.721965:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879625.721970:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879625.721971:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1788879625.721978:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879625.721987:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290182016/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1788879625.721988:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.721997:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879625.722011:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879625.722012:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879625.722016:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722022:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879625.722023:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879625.722029:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879625.722032:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879625.722035:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722037:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879625.722042:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879625.722047:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722048:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879625.722054:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879625.722060:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1788879625.722063:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290182400/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879625.722070:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879625.722080:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1788879625.722093:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1788879625.722096:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290182016:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879625.722099:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879625.722113:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722115:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722121:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879625.722124:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879625.722128:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879625.722133:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879625.722136:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879625.722139:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879625.722145:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879625.722147:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879625.722152:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879625.722154:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722157:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722167:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879625.722170:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722173:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879625.722175:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879625.722177:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879625.722179:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879625.722181:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879625.722184:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879625.722187:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879625.722190:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879625.722194:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290182144:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879625.722198:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879625.722201:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879625.722203:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879625.722205:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879625.722209:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722217:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879625.722218:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00000001:2.0:1788879625.722224:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879625.722242:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879625.722247:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879625.722250:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879625.722252:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722254:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879625.722258:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722263:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305237004 00000100:00000001:2.0:1788879625.722267:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305237545 00000100:00000001:2.0:1788879625.722270:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305237004 00000100:00000001:2.0:1788879625.722273:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305237545 00000100:00000001:2.0:1788879625.722276:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879625.722278:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722280:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879625.722285:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879625/real 1788879625] req@ffff94e508978700 x1875765290182144/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879625.722305:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879625.722306:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879625.722310:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879625.722312:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879625.722313:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722315:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879625.722318:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879625.722322:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290182144/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879625.722331:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722333:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879625.722336:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722338:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879625.722340:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722342:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879625.722361:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879625.722364:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879625.722369:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879625.722373:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879625.722380:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290182144:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879625.722387:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879625.722389:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879625.722392:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879625.722396:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879625.722400:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879625.722403:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879625.722409:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290182272:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879625.722415:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879625.722419:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879625.722422:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879625.722426:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879625.722432:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722443:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879625.722446:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00000001:2.0:1788879625.722453:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879625.722478:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879625.722487:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879625.722491:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879625.722493:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722495:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879625.722499:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722503:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305237002 00000100:00000001:2.0:1788879625.722506:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305237545 00000100:00000001:2.0:1788879625.722509:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305237002 00000100:00000001:2.0:1788879625.722512:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305237545 00000100:00000001:2.0:1788879625.722515:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879625.722517:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722520:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879625.722525:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879625/real 1788879625] req@ffff94e50897aa00 x1875765290182272/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879625.722535:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879625.722536:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879625.722539:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879625.722541:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879625.722543:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722545:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879625.722547:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879625.722579:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290182272/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879625.722589:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722592:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879625.722596:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722598:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879625.722600:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722602:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879625.722620:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879625.722624:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879625.722628:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879625.722631:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879625.722635:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290182272:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879625.722640:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879625.722641:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879625.722643:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879625.722646:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879625.722649:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879625.722651:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879625.722655:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290182400:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879625.722659:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879625.722662:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879625.722664:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879625.722666:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879625.722670:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722678:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879625.722680:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:2.0:1788879625.722684:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879625.722701:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879625.722706:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879625.722709:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879625.722711:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722713:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879625.722717:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722721:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305237002 00000100:00000001:2.0:1788879625.722724:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305237545 00000100:00000001:2.0:1788879625.722728:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305237002 00000100:00000001:2.0:1788879625.722730:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305237545 00000100:00000001:2.0:1788879625.722733:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879625.722735:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722738:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879625.722742:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879625/real 1788879625] req@ffff94e50897ad80 x1875765290182400/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879625.722751:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879625.722753:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879625.722756:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879625.722758:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879625.722760:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722762:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879625.722764:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879625.722769:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290182400/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879625.722776:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722779:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879625.722782:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722784:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879625.722786:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722788:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879625.722806:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879625.722809:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879625.722813:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879625.722816:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879625.722820:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290182400:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879625.722824:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722827:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879625.722829:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879625.722831:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879625.722834:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879625.722836:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879625.722837:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879625.722840:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879625.722842:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879625.722845:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879625.722846:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722848:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879625.722850:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879625.722852:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879625.722854:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879625.722855:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879625.722857:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879625.722859:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879625.722860:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879625.722863:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879625.722865:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722867:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879625.722868:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879625.722869:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879625.722871:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879625.722873:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879625.722874:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879625.722876:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879625.722877:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879625.722880:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879625.722881:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722884:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879625.722888:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879625.722890:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879625.722892:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879625.722894:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879625.785018:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879625.785025:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879625.785028:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879625.785034:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879625.785038:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879625.785042:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879625.785045:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879625.785048:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879625.785052:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879626.488989:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879626.488995:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879626.488999:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879626.489000:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879626.489018:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879626.489020:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879626.489023:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879626.808998:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879626.809007:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879626.809010:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879626.809017:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879626.809022:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879626.809028:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879626.809031:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879626.809035:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879626.809038:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879627.513040:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879627.513049:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879627.513051:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879627.513055:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879627.513061:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879627.513063:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879627.513066:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879627.513068:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879627.513072:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879627.513073:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879627.513151:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879627.513155:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879627.513175:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879627.832998:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879627.833004:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879627.833006:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879627.833011:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879627.833014:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879627.833029:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879627.833031:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879627.833033:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879627.833036:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879628.537011:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879628.537019:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879628.537026:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879628.537028:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879628.537056:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879628.537059:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879628.537064:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879628.857004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879628.857013:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879628.857027:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879628.857034:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879628.857039:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879628.857044:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879628.857048:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879628.857052:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879628.857056:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879629.561095:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879629.561105:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879629.561112:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879629.561114:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879629.561147:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879629.561151:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879629.561155:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879629.881044:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879629.881052:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879629.881055:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879629.881062:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879629.881066:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879629.881071:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879629.881075:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879629.881078:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879629.881082:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879630.585030:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879630.585038:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879630.585044:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879630.585046:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879630.585072:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879630.585075:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879630.585079:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879630.841048:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879630.841059:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879630.841063:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879630.841068:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879630.841075:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879630.841084:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10569 00000100:00080000:2.0:1788879630.841096:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879630.841099:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879630.841102:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879630.841111:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879630.841113:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879630.841117:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879630.841120:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879630.841122:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879630.841124:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879630.841128:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879630.841131:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879630.841134:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879630.841138:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879630.841141:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841144:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879630.841146:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879630.841162:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879630.841165:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841170:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879630.841174:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879630.841180:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841196:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879630.841210:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290182528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879630.841246:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879630.841253:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841281:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879630.841286:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879630.841290:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879630.841291:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879630.841305:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879630.841309:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879630.841313:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10569 00000100:00080000:2.0:1788879630.841319:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879630.841322:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879630.841341:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879630.841348:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879630.841349:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879630.841352:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879630.841357:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788879630.841357:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879630.841364:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879630.841370:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879630.841371:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1788879630.841379:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879630.841382:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879630.841385:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879630.841392:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879630.841393:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788879630.841402:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879630.841403:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879630.841415:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879630.841419:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879630.841420:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879630.841431:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:2.0:1788879630.841431:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879630.841445:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879630.841446:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841448:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879630.841458:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1788879630.841459:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290182528:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879630.841462:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841486:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879630.841488:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879630.841490:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841497:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.841509:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:2.0:1788879630.841509:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290182656/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879630.841555:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879630.841560:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879630.841578:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879630.841581:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879630.841587:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841588:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841599:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879630.841603:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879630.841607:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879630.841609:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879630.841610:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879630.841613:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879630.841618:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788879630.841618:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10569 00000100:00000001:1.0:1788879630.841627:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00080000:2.0:1788879630.841628:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879630.841634:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879630.841637:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879630.841639:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1788879630.841642:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879630.841645:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879630.841648:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879630.841650:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879630.841653:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879630.841655:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879630.841658:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879630.841662:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879630.841664:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879630.841668:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879630.841670:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841672:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879630.841675:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879630.841678:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841680:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879630.841686:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879630.841687:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841690:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879630.841693:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000800:00000100:1.0:1788879630.841697:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:2.0:1788879630.841699:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841709:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879630.841716:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290182784/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879630.841717:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879630.841724:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879630.841729:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.841736:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879630.841743:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841745:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879630.841750:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305242665 00000100:00000001:2.0:1788879630.841751:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841760:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879630.841763:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879630.841766:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879630.841768:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788879630.841769:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879630.841775:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879630.841779:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879630.841782:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10569 00000100:00080000:2.0:1788879630.841785:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879630.841787:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879630.841788:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879630.841790:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305242665 00000020:00000001:2.0:1788879630.841792:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879630.841796:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879630.841800:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879630.841801:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1788879630.841807:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879630.841809:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879630.841812:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879630.841816:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841817:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879630.841824:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1788879630.841826:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879630.841831:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879630.841836:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1788879630.841838:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879630/real 1788879630] req@ffff94e5045e5880 x1875765290182528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879630.841840:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879630.841849:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841856:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879630.841879:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879630.841884:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879630.841885:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788879630.841887:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879630.841890:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1788879630.841896:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879630.841898:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:2.0:1788879630.841900:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.841904:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879630.841906:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879630.841909:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879630.841910:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879630.841915:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1788879630.841972:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.841983:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879630.841991:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290182912/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879630.842022:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879630.842023:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879630.842031:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879630.842037:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879630.842044:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290182528/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879630.842089:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842094:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879630.842097:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842099:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879630.842101:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842104:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879630.842106:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879630.842110:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879630.842115:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879630.842118:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879630.842123:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290182528:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879630.842128:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842131:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879630.842133:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879630.842135:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879630.842140:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879630.842143:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879630.842145:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879630.842150:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879630.842151:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879630.842155:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879630.842156:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842159:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842166:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879630.842168:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.842171:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879630.842172:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879630.842174:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879630.842176:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879630.842178:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879630.842180:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879630.842183:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879630.842185:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879630.842188:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290182656:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879630.842192:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879630.842195:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879630.842197:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879630.842199:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879630.842202:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.842209:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879630.842211:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 00000100:00000001:1.0:1788879630.842216:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879630.842233:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879630.842255:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879630.842258:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879630.842260:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.842262:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879630.842266:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.842271:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305242124 00000100:00000001:1.0:1788879630.842275:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305242665 00000100:00000001:1.0:1788879630.842278:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305242124 00000100:00000001:1.0:1788879630.842281:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305242665 00000100:00000001:1.0:1788879630.842284:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879630.842287:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.842289:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879630.842294:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879630/real 1788879630] req@ffff94e5045e5180 x1875765290182656/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879630.842322:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879630.842324:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879630.842327:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879630.842329:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879630.842330:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842332:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879630.842333:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879630.842338:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290182656/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879630.842346:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842348:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879630.842350:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842352:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879630.842354:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842356:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879630.842398:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879630.842403:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879630.842409:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879630.842414:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879630.842420:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290182656:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879630.842427:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879630.842430:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879630.842433:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879630.842438:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879630.842441:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879630.842446:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879630.842452:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290182784:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879630.842474:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879630.842478:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879630.842480:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879630.842482:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879630.842486:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.842493:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879630.842495:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:1.0:1788879630.842500:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879630.842556:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879630.842562:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879630.842576:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879630.842578:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.842580:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879630.842583:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.842587:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305242122 00000100:00000001:1.0:1788879630.842590:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305242665 00000100:00000001:1.0:1788879630.842594:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305242122 00000100:00000001:1.0:1788879630.842598:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305242665 00000100:00000001:1.0:1788879630.842601:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879630.842604:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.842607:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879630.842613:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879630/real 1788879630] req@ffff94e5045e4a80 x1875765290182784/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879630.842627:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879630.842630:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879630.842634:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879630.842637:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879630.842639:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842642:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879630.842655:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879630.842661:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290182784/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879630.842683:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842686:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879630.842688:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842690:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879630.842692:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842694:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879630.842711:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879630.842714:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879630.842718:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879630.842720:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879630.842724:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290182784:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879630.842728:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879630.842730:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879630.842731:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879630.842733:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879630.842735:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879630.842737:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879630.842740:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290182912:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879630.842744:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879630.842746:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879630.842748:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879630.842750:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879630.842753:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.842759:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879630.842761:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:1.0:1788879630.842765:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879630.842779:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879630.842785:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879630.842787:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879630.842788:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.842791:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879630.842793:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.842797:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305242122 00000100:00000001:1.0:1788879630.842799:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305242665 00000100:00000001:1.0:1788879630.842802:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305242122 00000100:00000001:1.0:1788879630.842804:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305242665 00000100:00000001:1.0:1788879630.842806:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879630.842808:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.842809:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879630.842813:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879630/real 1788879630] req@ffff94e5045e7100 x1875765290182912/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879630.842821:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879630.842822:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879630.842824:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879630.842826:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879630.842827:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842829:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879630.842830:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879630.842834:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290182912/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879630.842840:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842843:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879630.842845:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842847:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879630.842848:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842850:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879630.842866:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879630.842868:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879630.842871:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879630.842873:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879630.842877:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290182912:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879630.842881:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842883:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879630.842884:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879630.842886:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879630.842889:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879630.842890:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879630.842891:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879630.842894:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879630.842895:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879630.842897:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879630.842898:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842900:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879630.842901:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879630.842903:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879630.842905:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879630.842906:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879630.842907:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879630.842908:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879630.842909:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879630.842911:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879630.842912:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842914:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879630.842915:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879630.842916:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879630.842944:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879630.842947:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879630.842948:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879630.842952:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879630.842954:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879630.842957:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879630.842958:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842962:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.842967:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879630.842981:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.842985:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879630.842987:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879630.905008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879630.905014:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879630.905016:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879630.905022:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879630.905025:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879630.905029:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.905032:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879630.905034:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879630.905037:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879631.609047:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879631.609056:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879631.609065:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879631.609067:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879631.609154:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879631.609161:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879631.609167:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879631.929008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879631.929019:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879631.929022:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879631.929029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879631.929034:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879631.929040:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879631.929044:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879631.929047:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879631.929051:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879632.633069:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879632.633079:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879632.633089:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879632.633092:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879632.633144:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879632.633150:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879632.633155:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879632.953011:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879632.953019:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879632.953022:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879632.953028:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879632.953033:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879632.953039:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879632.953042:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879632.953046:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879632.953049:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879633.656996:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879633.657004:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879633.657010:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879633.657011:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879633.657032:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879633.657035:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879633.657037:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879633.977049:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879633.977069:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879633.977072:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879633.977079:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879633.977083:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879633.977088:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879633.977091:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879633.977095:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879633.977098:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879634.681078:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879634.681089:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879634.681097:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879634.681099:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879634.681140:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879634.681143:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879634.681153:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879635.000980:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879635.000990:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879635.000992:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879635.001000:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879635.001005:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879635.001010:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.001014:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879635.001017:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.001020:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879635.705096:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879635.705107:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879635.705123:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879635.705131:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879635.705135:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879635.705142:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879635.705147:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879635.705152:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879635.705156:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879635.705160:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879635.705255:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879635.705261:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879635.705266:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879635.961145:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879635.961163:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879635.961168:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879635.961174:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879635.961182:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879635.961192:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10574 00000100:00080000:0.0:1788879635.961206:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879635.961210:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879635.961213:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879635.961222:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879635.961225:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879635.961230:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879635.961233:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879635.961236:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879635.961238:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879635.961243:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879635.961246:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879635.961250:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879635.961255:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879635.961259:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.961264:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879635.961267:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879635.961282:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879635.961285:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.961289:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879635.961293:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879635.961299:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.961315:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879635.961328:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290183040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879635.961342:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879635.961347:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.961371:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879635.961376:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879635.961380:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879635.961382:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879635.961384:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879635.961388:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879635.961391:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10574 00000100:00080000:0.0:1788879635.961395:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879635.961397:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879635.961399:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879635.961403:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879635.961405:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879635.961407:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879635.961410:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879635.961412:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879635.961414:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879635.961418:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879635.961420:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879635.961422:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879635.961430:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879635.961432:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.961435:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879635.961440:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879635.961444:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.961447:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879635.961451:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879635.961453:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.961455:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879635.961458:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879635.961461:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.961468:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879635.961511:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290183168/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879635.961526:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879635.961527:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879635.961531:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.961539:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879635.961549:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879635.961551:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1788879635.961560:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879635.961563:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879635.961565:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879635.961577:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879635.961578:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879635.961585:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879635.961590:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879635.961596:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10574 00000100:00080000:0.0:1788879635.961610:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788879635.961614:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788879635.961616:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879635.961624:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879635.961627:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788879635.961636:0:178195:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1788879635.961640:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879635.961641:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879635.961657:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879635.961668:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00100000:1.0:1788879635.961669:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290183040:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879635.961676:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879635.961684:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879635.961686:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879635.961701:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1788879635.961703:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788879635.961710:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.961712:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879635.961720:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879635.961721:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879635.961737:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879635.961741:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.961744:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879635.961747:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788879635.961748:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879635.961763:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.961769:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879635.961775:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879635.961777:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.961779:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879635.961782:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879635.961784:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.961792:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.961799:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00100000:0.0:1788879635.961801:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290183296/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879635.961808:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:0.0:1788879635.961830:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879635.961834:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.961834:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879635.961851:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879635.961856:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879635.961861:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879635.961863:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879635.961865:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879635.961872:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879635.961877:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10574 00000100:00080000:0.0:1788879635.961897:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879635.961900:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000800:00000100:1.0:1788879635.961988:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879635.962003:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879635.962010:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:0.0:1788879635.962012:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879635.962017:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879635.962021:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879635.962022:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879635.962029:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879635.962030:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879635.962036:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879635.962045:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879635.962051:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.962054:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879635.962061:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305247786 00000100:00080000:0.0:1788879635.962076:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879635.962083:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788879635.962085:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879635.962096:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879635.962098:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305247786 00000100:00000001:1.0:1788879635.962110:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1788879635.962112:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879635.962126:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879635.962129:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.962132:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.962148:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879635.962150:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879635.962155:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879635.962162:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879635/real 1788879635] req@ffff94e508978e00 x1875765290183040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1788879635.962165:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.962178:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879635.962195:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879635.962201:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.962214:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879635.962224:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879635.962234:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879635.962237:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879635.962245:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879635.962249:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.962252:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879635.962267:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879635.962275:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879635.962276:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290183424/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879635.962293:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879635.962300:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879635.962319:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1788879635.962328:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290183040/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879635.962330:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879635.962342:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.962363:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.962369:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879635.962374:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.962376:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879635.962378:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.962382:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879635.962385:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879635.962392:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879635.962398:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879635.962402:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879635.962409:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290183040:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879635.962419:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879635.962423:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879635.962427:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879635.962433:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879635.962437:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879635.962440:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879635.962444:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290183168:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879635.962449:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879635.962452:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879635.962456:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879635.962458:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879635.962463:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.962471:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879635.962473:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:1.0:1788879635.962503:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879635.962531:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879635.962539:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879635.962544:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879635.962547:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.962550:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879635.962556:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.962564:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305247244 00000100:00000001:1.0:1788879635.962570:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305247786 00000100:00000001:1.0:1788879635.962575:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305247244 00000100:00000001:1.0:1788879635.962580:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305247786 00000100:00000001:1.0:1788879635.962585:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879635.962588:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.962593:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879635.962602:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879635/real 1788879635] req@ffff94e50897a300 x1875765290183168/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879635.962618:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879635.962619:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879635.962624:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879635.962626:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879635.962628:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.962630:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879635.962632:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879635.962638:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290183168/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879635.962647:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.962650:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879635.962654:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.962656:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879635.962658:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.962660:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879635.962678:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879635.962681:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879635.962686:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879635.962689:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879635.962693:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290183168:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879635.962698:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.962701:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879635.962703:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879635.962707:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879635.962714:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879635.962717:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879635.962720:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879635.962727:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879635.962730:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879635.962738:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879635.962740:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.962744:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879635.962746:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879635.962749:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879635.962752:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879635.962754:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879635.962756:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879635.962760:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879635.962763:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879635.962767:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879635.962770:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.962775:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.962788:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879635.962792:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.962797:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879635.962801:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879635.962804:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879635.962807:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879635.962811:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879635.962817:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879635.962820:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879635.962824:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879635.962831:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290183296:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879635.962836:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879635.962839:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879635.962841:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879635.962844:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879635.962848:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.962858:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879635.962860:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:1.0:1788879635.962865:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879635.962883:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879635.962890:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879635.962893:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879635.962895:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.962897:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879635.962900:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.962941:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305247243 00000100:00000001:1.0:1788879635.962945:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305247786 00000100:00000001:1.0:1788879635.962948:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305247243 00000100:00000001:1.0:1788879635.962951:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305247786 00000100:00000001:1.0:1788879635.962954:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879635.962956:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.962959:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879635.962964:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879635/real 1788879635] req@ffff94e508979c00 x1875765290183296/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879635.962974:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879635.962976:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879635.962980:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879635.962982:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879635.962984:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.962987:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879635.962990:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879635.962994:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290183296/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879635.963003:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.963006:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879635.963009:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.963011:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879635.963012:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.963014:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879635.963029:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879635.963032:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879635.963036:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879635.963038:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879635.963042:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290183296:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879635.963046:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879635.963048:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879635.963050:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879635.963053:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879635.963055:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879635.963058:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879635.963062:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290183424:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879635.963066:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879635.963068:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879635.963070:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879635.963072:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879635.963075:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.963081:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879635.963083:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:1.0:1788879635.963087:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879635.963100:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879635.963106:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879635.963108:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879635.963110:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.963113:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879635.963116:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.963120:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305247243 00000100:00000001:1.0:1788879635.963124:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305247786 00000100:00000001:1.0:1788879635.963127:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305247243 00000100:00000001:1.0:1788879635.963130:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305247786 00000100:00000001:1.0:1788879635.963132:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879635.963135:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.963137:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879635.963141:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879635/real 1788879635] req@ffff94e50897bb80 x1875765290183424/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879635.963150:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879635.963152:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879635.963154:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879635.963156:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879635.963157:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.963159:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879635.963161:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879635.963165:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290183424/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879635.963174:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.963177:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879635.963180:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.963182:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879635.963184:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.963186:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879635.963200:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879635.963203:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879635.963208:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879635.963210:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879635.963215:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290183424:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879635.963219:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.963221:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879635.963223:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879635.963225:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879635.963227:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879635.963229:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879635.963230:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879635.963233:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879635.963235:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879635.963237:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879635.963239:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.963241:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879635.963243:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879635.963244:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879635.963246:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879635.963248:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879635.963249:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879635.963251:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879635.963252:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879635.963255:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879635.963257:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.963259:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879635.963263:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879635.963265:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879635.963267:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879635.963269:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879636.025027:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879636.025038:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879636.025041:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879636.025049:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879636.025054:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879636.025059:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879636.025062:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879636.025066:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879636.025070:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879636.729076:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879636.729087:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879636.729097:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879636.729100:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879636.729155:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879636.729160:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879636.729166:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879637.049054:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879637.049063:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879637.049067:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879637.049072:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879637.049076:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879637.049082:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879637.049086:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879637.049089:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879637.049092:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879637.753065:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879637.753073:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879637.753076:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879637.753081:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879637.753086:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879637.753088:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879637.753091:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879637.753093:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879637.753097:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879637.753099:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879637.753120:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879637.753123:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879637.753147:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879638.072995:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879638.073003:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879638.073006:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879638.073011:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879638.073015:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879638.073020:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879638.073023:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879638.073026:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879638.073029:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879638.777014:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879638.777019:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879638.777025:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879638.777026:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879638.777060:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879638.777063:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879638.777066:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879639.097030:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879639.097039:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879639.097042:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879639.097049:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879639.097054:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879639.097060:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879639.097063:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879639.097067:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879639.097071:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879639.801057:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879639.801067:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879639.801078:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879639.801081:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879639.801207:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879639.801213:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879639.801218:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879640.121002:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879640.121010:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879640.121013:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879640.121020:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879640.121025:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879640.121030:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879640.121034:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879640.121037:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879640.121041:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879640.825091:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879640.825102:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879640.825113:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879640.825115:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879640.825198:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879640.825202:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879640.825215:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879641.081047:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879641.081065:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879641.081070:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879641.081077:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879641.081085:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879641.081098:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10579 00000100:00080000:0.0:1788879641.081116:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879641.081123:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879641.081140:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879641.081154:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879641.081158:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879641.081163:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879641.081166:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879641.081169:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879641.081171:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879641.081178:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879641.081182:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879641.081186:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879641.081191:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879641.081195:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879641.081199:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879641.081202:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879641.081216:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879641.081218:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879641.081223:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879641.081226:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879641.081232:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879641.081247:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879641.081259:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290183552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879641.081273:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879641.081291:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879641.081325:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879641.081330:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879641.081335:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879641.081337:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879641.081339:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879641.081343:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879641.081346:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10579 00000100:00080000:0.0:1788879641.081352:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879641.081355:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879641.081357:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879641.081361:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879641.081364:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879641.081366:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879641.081370:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879641.081373:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879641.081376:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879641.081381:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879641.081385:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879641.081387:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879641.081395:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879641.081398:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879641.081400:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879641.081403:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:0.0:1788879641.081405:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879641.081413:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.081416:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879641.081417:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879641.081424:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879641.081427:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1788879641.081429:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879641.081435:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879641.081437:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1788879641.081438:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879641.081467:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879641.081470:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879641.081484:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788879641.081485:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879641.081504:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788879641.081507:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979180 x1875765290183680/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879641.081512:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879641.081532:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290183552:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879641.081533:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879641.081542:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.081549:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879641.081555:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879641.081558:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879641.081562:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879641.081563:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879641.081571:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879641.081571:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879641.081580:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879641.081585:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879641.081588:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879641.081602:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879641.081607:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10579 00000100:00000001:1.0:1788879641.081611:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1788879641.081613:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879641.081618:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 00000100:00000001:0.0:1788879641.081620:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879641.081625:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879641.081630:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1788879641.081632:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879641.081637:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879641.081640:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879641.081644:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879641.081647:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879641.081649:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879641.081654:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879641.081658:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879641.081660:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879641.081664:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879641.081667:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879641.081670:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879641.081673:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879641.081676:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879641.081679:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879641.081687:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:0.0:1788879641.081688:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879641.081694:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879641.081698:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879641.081700:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879641.081703:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879641.081710:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.081712:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00100000:0.0:1788879641.081720:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290183808/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879641.081721:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879641.081729:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.081738:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:0.0:1788879641.081747:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879641.081749:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879641.081752:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.081757:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305252905 00000100:00000001:0.0:1788879641.081759:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879641.081767:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879641.081772:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879641.081774:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879641.081777:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879641.081779:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788879641.081780:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879641.081790:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10579 00000100:00080000:0.0:1788879641.081800:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879641.081803:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879641.081805:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305252905 00000100:00000001:0.0:1788879641.081806:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879641.081819:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879641.081821:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:0.0:1788879641.081822:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879641.081829:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879641.081832:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1788879641.081834:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879641.081844:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879641.081845:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879641.081850:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879641.081853:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:0.0:1788879641.081860:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879641.081868:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879641.081871:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879641.081874:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879641.081877:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879641.081879:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879641.081881:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879641.081883:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879641.081886:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879641/real 1788879641] req@ffff94e508978380 x1875765290183552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879641.081887:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879641.081896:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879641.081899:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879641.081948:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879641.081953:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879641.081957:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879641.081968:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879641.081974:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879641.081982:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:0.0:1788879641.081983:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290183936/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879641.081989:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879641.081997:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879641.082003:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082013:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879641.082019:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879641.082022:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879641.082027:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879641.082034:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290183552/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879641.082036:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.082054:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082061:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879641.082065:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082067:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879641.082070:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082073:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879641.082077:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879641.082082:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879641.082088:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879641.082092:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879641.082096:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290183552:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879641.082102:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082105:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879641.082107:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879641.082110:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879641.082115:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879641.082117:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879641.082120:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879641.082135:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879641.082137:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879641.082141:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879641.082143:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082147:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082158:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879641.082162:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.082166:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879641.082169:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879641.082172:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879641.082175:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879641.082178:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879641.082182:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879641.082186:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879641.082190:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879641.082197:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290183680:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879641.082205:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879641.082209:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879641.082212:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879641.082215:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879641.082221:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.082234:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879641.082237:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 00000100:00000001:1.0:1788879641.082256:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879641.082280:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879641.082288:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879641.082292:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879641.082295:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.082299:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879641.082304:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.082312:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305252364 00000100:00000001:1.0:1788879641.082318:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305252905 00000100:00000001:1.0:1788879641.082322:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305252364 00000100:00000001:1.0:1788879641.082327:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305252905 00000100:00000001:1.0:1788879641.082331:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879641.082335:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.082339:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879641.082349:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879641/real 1788879641] req@ffff94e508979180 x1875765290183680/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879641.082368:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879641.082371:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879641.082377:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879641.082380:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879641.082383:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082388:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879641.082391:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879641.082400:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290183680/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879641.082414:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082418:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879641.082423:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082426:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879641.082429:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082432:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879641.082491:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879641.082498:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879641.082507:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879641.082510:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879641.082515:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290183680:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879641.082521:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879641.082523:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879641.082525:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879641.082528:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879641.082531:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879641.082533:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879641.082537:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290183808:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879641.082541:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879641.082544:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879641.082546:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879641.082549:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879641.082553:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.082562:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879641.082564:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000100:00000001:1.0:1788879641.082569:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879641.082588:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879641.082601:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879641.082605:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879641.082607:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.082609:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879641.082613:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.082617:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305252362 00000100:00000001:1.0:1788879641.082621:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305252905 00000100:00000001:1.0:1788879641.082623:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305252362 00000100:00000001:1.0:1788879641.082627:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305252905 00000100:00000001:1.0:1788879641.082630:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879641.082632:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.082634:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879641.082639:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879641/real 1788879641] req@ffff94e50897b800 x1875765290183808/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879641.082649:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879641.082650:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879641.082653:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879641.082655:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879641.082657:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082659:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879641.082661:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879641.082666:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290183808/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879641.082674:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082677:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879641.082679:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082681:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879641.082683:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082685:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879641.082703:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879641.082706:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879641.082709:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879641.082713:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879641.082717:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290183808:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879641.082721:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879641.082723:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879641.082725:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879641.082727:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879641.082729:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879641.082732:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879641.082735:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290183936:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879641.082739:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879641.082741:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879641.082743:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879641.082745:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879641.082749:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.082754:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879641.082756:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:1.0:1788879641.082760:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879641.082771:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879641.082775:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879641.082778:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879641.082780:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.082782:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879641.082785:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.082789:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305252362 00000100:00000001:1.0:1788879641.082803:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305252905 00000100:00000001:1.0:1788879641.082806:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305252362 00000100:00000001:1.0:1788879641.082809:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305252905 00000100:00000001:1.0:1788879641.082812:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879641.082814:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.082816:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879641.082820:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879641/real 1788879641] req@ffff94e50897bb80 x1875765290183936/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879641.082839:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879641.082840:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879641.082843:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879641.082845:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879641.082846:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082848:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879641.082850:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879641.082854:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290183936/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879641.082862:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082865:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879641.082867:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082869:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879641.082870:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082872:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879641.082889:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879641.082892:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879641.082895:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879641.082898:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879641.082901:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290183936:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879641.082938:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082940:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879641.082942:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879641.082944:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879641.082947:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879641.082949:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879641.082951:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879641.082954:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879641.082956:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879641.082960:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879641.082962:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082965:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879641.082966:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879641.082968:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879641.082969:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879641.082971:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879641.082973:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879641.082975:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879641.082976:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879641.082979:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879641.082981:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082982:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879641.082984:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879641.082985:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879641.082987:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879641.082989:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879641.082990:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879641.082992:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879641.082993:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879641.082995:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879641.082996:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.082998:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.083003:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879641.083005:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.083007:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879641.083009:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879641.145000:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879641.145009:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879641.145012:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879641.145018:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879641.145022:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879641.145027:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.145030:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879641.145033:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879641.145037:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879641.849049:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879641.849079:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879641.849091:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879641.849094:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879641.849171:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879641.849174:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879641.849178:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879642.168997:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879642.169005:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879642.169008:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879642.169013:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879642.169017:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879642.169021:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879642.169024:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879642.169028:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879642.169030:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879642.873035:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879642.873043:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879642.873049:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879642.873051:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879642.873097:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879642.873100:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879642.873104:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879643.193019:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879643.193028:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879643.193031:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879643.193038:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879643.193043:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879643.193048:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879643.193052:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879643.193055:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879643.193059:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879643.897074:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879643.897085:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879643.897097:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879643.897100:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879643.897190:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879643.897199:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879643.897207:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879644.217029:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879644.217038:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879644.217041:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879644.217059:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879644.217064:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879644.217070:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879644.217073:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879644.217077:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879644.217080:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879644.921018:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879644.921024:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879644.921029:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879644.921031:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879644.921087:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879644.921093:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879644.921099:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879645.240967:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879645.240974:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879645.240976:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879645.240980:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879645.240983:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879645.240987:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879645.240989:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879645.240991:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879645.241010:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879645.945041:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879645.945049:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879645.945056:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879645.945061:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879645.945064:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879645.945068:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879645.945072:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879645.945074:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879645.945077:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879645.945079:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879645.945139:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879645.945143:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879645.945147:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879646.201090:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879646.201107:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879646.201113:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879646.201120:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879646.201127:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879646.201139:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10585 00000100:00080000:2.0:1788879646.201157:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879646.201164:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879646.201168:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879646.201181:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879646.201188:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879646.201193:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879646.201199:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879646.201203:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879646.201206:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879646.201215:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879646.201222:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879646.201228:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879646.201234:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879646.201241:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879646.201246:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879646.201251:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879646.201278:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879646.201282:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879646.201288:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879646.201294:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879646.201301:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879646.201320:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879646.201337:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290184064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879646.201358:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879646.201365:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879646.201393:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879646.201401:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879646.201432:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879646.201447:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879646.201451:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879646.201460:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879646.201466:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10585 00000100:00080000:2.0:1788879646.201475:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879646.201481:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879646.201485:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879646.201494:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879646.201497:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879646.201511:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879646.201517:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879646.201522:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879646.201525:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879646.201533:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879646.201539:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879646.201541:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.201553:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879646.201559:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879646.201563:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1788879646.201578:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879646.201579:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879646.201594:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879646.201598:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879646.201600:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879646.201610:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879646.201614:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879646.201624:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:1.0:1788879646.201628:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290184064:192.168.203.129@tcp:250:kworker.0 00000008:00100000:2.0:1788879646.201635:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879646.201639:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879646.201650:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:2.0:1788879646.201651:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879646.201656:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879646.201664:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879646.201665:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879646.201691:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879646.201695:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879646.201698:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879646.201700:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879646.201707:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879646.201711:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879646.201727:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.201730:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879646.201734:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:1.0:1788879646.201740:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1788879646.201744:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290184192/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879646.201765:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879646.201770:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879646.201778:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879646.201784:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879646.201793:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879646.201797:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000800:00000100:1.0:1788879646.201807:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788879646.201809:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879646.201819:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879646.201828:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10585 00000100:00000001:1.0:1788879646.201829:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879646.201841:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879646.201845:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879646.201848:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879646.201849:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879646.201860:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879646.201861:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879646.201869:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879646.201875:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305258025 00000020:00000001:2.0:1788879646.201885:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879646.201891:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879646.201896:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879646.201898:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788879646.201915:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305258025 00000020:00000001:2.0:1788879646.201965:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879646.201972:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879646.201979:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879646.201983:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879646.201991:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788879646.201993:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879646.201998:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.202009:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879646.202010:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879646.202020:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1788879646.202029:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879646/real 1788879646] req@ffff94e5045e5c00 x1875765290184064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879646.202032:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879646.202050:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879646.202061:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879646.202077:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000400:00000001:1.0:1788879646.202078:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879646.202082:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879646.202089:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000008:00000001:2.0:1788879646.202093:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.202095:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879646.202102:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879646.202105:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.202110:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879646.202115:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879646.202122:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290184064/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879646.202123:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879646.202133:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879646.202154:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879646.202155:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879646.202167:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879646.202169:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879646.202178:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.202181:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1788879646.202181:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.202186:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.202197:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879646.202199:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879646.202203:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788879646.202205:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879646.202210:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879646.202220:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879646.202233:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290184064:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1788879646.202236:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4380 x1875765290184320/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879646.202240:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.202248:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879646.202269:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879646.202279:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879646.202288:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1788879646.202292:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000020:00000001:1.0:1788879646.202293:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879646.202299:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:2.0:1788879646.202306:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879646.202308:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879646.202315:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879646.202319:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.202323:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879646.202328:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879646.202330:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879646.202334:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879646.202349:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879646.202351:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879646.202355:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879646.202357:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879646.202364:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879646.202365:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879646.202370:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879646.202375:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879646.202380:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879646.202381:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879646.202388:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879646.202394:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1788879646.202396:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10585 02000000:00000001:1.0:1788879646.202399:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879646.202438:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879646.202443:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1788879646.202446:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290184192:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879646.202465:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879646.202472:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879646.202476:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879646.202486:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:2.0:1788879646.202488:0:178195:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1788879646.202492:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879646.202495:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1788879646.202499:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1788879646.202504:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788879646.202506:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879646.202518:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879646.202522:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879646.202532:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879646.202533:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:2.0:1788879646.202545:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879646.202549:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1788879646.202566:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879646.202572:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879646.202578:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:1.0:1788879646.202585:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:2.0:1788879646.202588:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879646.202594:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:2.0:1788879646.202599:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.202601:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879646.202608:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879646.202611:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879646.202614:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879646.202622:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1788879646.202623:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879646.202631:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305257484 00000008:00000001:2.0:1788879646.202636:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.202638:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305258025 00000100:00000001:2.0:1788879646.202655:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.202656:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305257484 00000100:00000001:1.0:1788879646.202671:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305258025 00000100:00000001:1.0:1788879646.202677:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879646.202679:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879646.202681:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879646.202699:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.202705:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879646.202713:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1788879646.202719:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879646/real 1788879646] req@ffff94e5045e5500 x1875765290184192/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879646.202720:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879646.202729:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879646.202748:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879646.202750:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879646.202753:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879646.202755:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879646.202757:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.202758:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.202770:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879646.202775:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879646.202796:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290184448/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879646.202804:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290184192/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879646.202898:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879646.202901:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.202908:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879646.202915:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.202920:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879646.202922:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.202926:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879646.202946:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.202950:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879646.202958:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879646.202960:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879646.202994:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879646.202997:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879646.203004:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290184192:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879646.203009:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879646.203011:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879646.203013:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879646.203016:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879646.203019:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879646.203021:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879646.203024:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290184320:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879646.203028:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879646.203031:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879646.203032:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879646.203034:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879646.203038:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.203045:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879646.203047:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074549120 : -117763635002496 : ffff94e5045e4380) 00000100:00000001:1.0:1788879646.203051:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879646.203064:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879646.203069:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879646.203071:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879646.203073:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.203074:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879646.203077:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.203081:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305257483 00000100:00000001:1.0:1788879646.203084:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305258025 00000100:00000001:1.0:1788879646.203086:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305257483 00000100:00000001:1.0:1788879646.203089:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305258025 00000100:00000001:1.0:1788879646.203091:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879646.203093:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.203095:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879646.203098:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879646/real 1788879646] req@ffff94e5045e4380 x1875765290184320/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879646.203106:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879646.203107:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879646.203109:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879646.203110:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879646.203112:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.203114:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879646.203115:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879646.203118:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4380 x1875765290184320/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879646.203142:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.203144:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879646.203147:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.203149:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879646.203150:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.203152:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879646.203164:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879646.203166:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879646.203169:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879646.203172:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879646.203175:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290184320:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879646.203179:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.203181:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879646.203183:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879646.203184:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879646.203187:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879646.203189:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879646.203190:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879646.203192:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879646.203194:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879646.203196:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879646.203197:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.203199:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879646.203200:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879646.203202:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879646.203204:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879646.203205:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879646.203207:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879646.203209:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879646.203210:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879646.203212:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879646.203213:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.203215:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.203218:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879646.203219:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.203221:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879646.203223:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879646.203224:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879646.203225:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879646.203227:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879646.203229:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879646.203230:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879646.203232:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879646.203235:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290184448:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879646.203239:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879646.203241:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879646.203243:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879646.203244:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879646.203247:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.203251:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879646.203253:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000100:00000001:1.0:1788879646.203257:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879646.203268:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879646.203272:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879646.203274:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879646.203275:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.203277:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879646.203279:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.203283:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305257483 00000100:00000001:1.0:1788879646.203285:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305258025 00000100:00000001:1.0:1788879646.203288:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305257483 00000100:00000001:1.0:1788879646.203290:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305258025 00000100:00000001:1.0:1788879646.203292:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879646.203294:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.203296:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879646.203299:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879646/real 1788879646] req@ffff94e5045e7800 x1875765290184448/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879646.203306:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879646.203307:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879646.203309:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879646.203310:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879646.203311:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.203313:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879646.203315:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879646.203318:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290184448/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879646.203326:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.203327:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879646.203329:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.203331:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879646.203332:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.203333:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879646.203345:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879646.203348:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879646.203351:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879646.203353:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879646.203357:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290184448:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879646.203360:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.203362:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879646.203363:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879646.203364:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879646.203366:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879646.203367:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879646.203368:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879646.203369:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879646.203370:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879646.203372:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879646.203373:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.203375:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.203377:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879646.203379:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.203381:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879646.203383:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879646.265054:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879646.265076:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879646.265080:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879646.265090:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879646.265097:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879646.265118:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.265122:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879646.265127:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879646.265131:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879646.969090:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879646.969099:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879646.969106:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879646.969107:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879646.969138:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879646.969142:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879646.969145:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879647.288955:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879647.288960:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879647.288962:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879647.288965:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879647.288968:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879647.288971:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879647.288973:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879647.288976:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879647.288978:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879647.992995:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879647.993020:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879647.993023:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879647.993027:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879647.993045:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879647.993048:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879647.993050:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879647.993054:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879647.993058:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879647.993061:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879647.993175:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879647.993181:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879647.993186:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879648.313049:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879648.313059:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879648.313063:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879648.313071:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879648.313077:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879648.313083:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879648.313088:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879648.313093:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879648.313097:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879649.017034:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879649.017043:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879649.017050:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879649.017052:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879649.017154:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879649.017163:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879649.017171:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879649.337026:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879649.337035:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879649.337038:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879649.337046:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879649.337053:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879649.337061:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879649.337066:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879649.337072:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879649.337077:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879650.041062:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879650.041070:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879650.041078:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879650.041080:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879650.041109:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879650.041113:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879650.041117:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879650.361053:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879650.361082:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879650.361087:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879650.361095:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879650.361102:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879650.361108:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879650.361114:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879650.361119:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879650.361124:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879651.065047:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879651.065055:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879651.065062:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879651.065064:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879651.065122:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879651.065125:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879651.065129:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879651.321150:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879651.321165:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879651.321170:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879651.321176:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879651.321182:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879651.321192:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10590 00000100:00080000:3.0:1788879651.321210:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879651.321214:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879651.321216:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879651.321229:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879651.321233:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879651.321238:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879651.321241:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879651.321244:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879651.321246:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879651.321254:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879651.321258:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879651.321262:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879651.321267:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879651.321272:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.321275:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879651.321278:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879651.321297:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879651.321299:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.321303:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879651.321307:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879651.321315:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.321333:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879651.321348:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290184576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879651.321363:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879651.321394:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.321419:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879651.321424:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879651.321429:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879651.321432:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879651.321434:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879651.321438:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879651.321442:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10590 00000100:00080000:3.0:1788879651.321447:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879651.321450:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879651.321452:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879651.321458:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879651.321460:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879651.321463:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879651.321466:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879651.321470:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879651.321474:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879651.321479:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879651.321482:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879651.321485:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879651.321493:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879651.321496:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.321499:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879651.321500:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879651.321509:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879651.321512:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00000001:3.0:1788879651.321517:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.321522:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879651.321523:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.321533:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788879651.321534:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879651.321539:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.321544:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.321545:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879651.321549:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879651.321554:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879651.321557:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879651.321562:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879651.321564:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879651.321573:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879651.321576:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290184704/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879651.321587:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290184576:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879651.321602:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879651.321605:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879651.321608:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.321616:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879651.321617:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1788879651.321620:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879651.321623:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879651.321630:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:1.0:1788879651.321632:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879651.321636:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879651.321641:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1788879651.321644:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.321645:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879651.321656:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10590 00000100:00080000:3.0:1788879651.321661:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879651.321664:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879651.321666:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879651.321670:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879651.321672:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879651.321675:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879651.321677:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1788879651.321679:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879651.321685:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:3.0:1788879651.321687:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879651.321692:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879651.321697:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879651.321699:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879651.321705:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879651.321707:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879651.321711:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879651.321714:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.321716:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879651.321719:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879651.321722:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.321725:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879651.321731:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879651.321732:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.321734:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879651.321737:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879651.321740:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.321749:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879651.321754:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290184832/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788879651.321768:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879651.321778:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879651.321782:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.321786:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879651.321790:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879651.321796:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:3.0:1788879651.321797:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879651.321805:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879651.321807:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879651.321810:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879651.321812:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879651.321816:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879651.321822:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10590 00000100:00000001:1.0:1788879651.321823:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:3.0:1788879651.321833:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879651.321836:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.321838:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879651.321845:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879651.321848:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305263145 00000020:00000001:3.0:1788879651.321851:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879651.321855:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879651.321860:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879651.321863:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879651.321866:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879651.321868:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879651.321871:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788879651.321873:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879651.321882:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879651.321884:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879651.321888:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879651.321891:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.321893:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879651.321895:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879651.321898:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.321961:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.321964:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305263146 02000000:00000001:3.0:1788879651.321969:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879651.321973:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.321976:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879651.321977:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1788879651.321979:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879651.321984:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.321989:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879651.321995:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.321996:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.322008:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1788879651.322013:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290184960/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879651.322024:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879651/real 1788879651] req@ffff94e50449f480 x1875765290184576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879651.322036:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879651.322044:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879651.322054:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879651.322060:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879651.322066:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879651.322073:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879651.322077:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879651.322079:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322084:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879651.322087:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879651.322095:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290184576/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879651.322107:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322111:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879651.322115:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322118:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879651.322120:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322123:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879651.322126:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879651.322130:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879651.322136:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879651.322139:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879651.322144:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290184576:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879651.322150:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322153:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879651.322155:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879651.322159:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879651.322164:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879651.322167:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879651.322170:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879651.322176:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879651.322179:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879651.322183:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879651.322185:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322187:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322197:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879651.322199:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.322202:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879651.322204:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879651.322207:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879651.322208:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879651.322211:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879651.322214:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879651.322216:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879651.322219:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879651.322223:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290184704:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879651.322227:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879651.322229:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879651.322231:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879651.322233:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879651.322238:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.322247:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879651.322249:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:1.0:1788879651.322256:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879651.322272:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879651.322277:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879651.322280:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879651.322282:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.322284:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879651.322288:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.322293:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305262604 00000100:00000001:1.0:1788879651.322297:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305263145 00000100:00000001:1.0:1788879651.322300:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305262604 00000100:00000001:1.0:1788879651.322303:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305263146 00000100:00000001:1.0:1788879651.322305:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879651.322307:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.322310:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879651.322315:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879651/real 1788879651] req@ffff94e50449c700 x1875765290184704/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879651.322325:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879651.322326:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879651.322329:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879651.322331:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879651.322333:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322335:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879651.322337:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879651.322342:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290184704/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879651.322350:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322352:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879651.322355:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322357:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879651.322359:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322361:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879651.322405:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879651.322412:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879651.322419:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879651.322424:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879651.322431:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290184704:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879651.322438:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879651.322441:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879651.322444:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879651.322449:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879651.322452:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879651.322456:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879651.322462:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290184832:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879651.322469:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879651.322473:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879651.322476:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879651.322480:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879651.322486:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.322500:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879651.322503:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:1.0:1788879651.322510:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879651.322532:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879651.322538:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879651.322542:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879651.322544:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.322546:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879651.322550:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.322554:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305262602 00000100:00000001:1.0:1788879651.322557:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305263145 00000100:00000001:1.0:1788879651.322560:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305262602 00000100:00000001:1.0:1788879651.322563:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305263146 00000100:00000001:1.0:1788879651.322567:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879651.322569:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.322572:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879651.322577:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879651/real 1788879651] req@ffff94e50449e300 x1875765290184832/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879651.322588:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879651.322590:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879651.322592:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879651.322594:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879651.322596:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322598:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879651.322600:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879651.322605:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290184832/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879651.322614:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322616:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879651.322619:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322621:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879651.322622:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322625:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879651.322637:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879651.322640:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879651.322644:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879651.322647:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879651.322650:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290184832:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879651.322654:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879651.322656:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879651.322658:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879651.322661:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879651.322664:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879651.322666:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879651.322671:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290184960:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879651.322675:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879651.322678:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879651.322680:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879651.322683:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879651.322686:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.322692:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879651.322693:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:1.0:1788879651.322698:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879651.322710:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879651.322714:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879651.322716:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879651.322718:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.322720:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879651.322723:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.322727:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305262602 00000100:00000001:1.0:1788879651.322730:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305263145 00000100:00000001:1.0:1788879651.322733:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305262602 00000100:00000001:1.0:1788879651.322736:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305263146 00000100:00000001:1.0:1788879651.322739:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879651.322741:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.322743:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879651.322748:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879651/real 1788879651] req@ffff94e50449c000 x1875765290184960/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879651.322756:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879651.322758:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879651.322760:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879651.322762:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879651.322763:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322765:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879651.322767:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879651.322772:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290184960/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879651.322789:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322792:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879651.322795:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322796:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879651.322798:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322800:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879651.322813:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879651.322815:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879651.322819:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879651.322821:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879651.322825:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290184960:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879651.322829:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322832:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879651.322834:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879651.322836:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879651.322838:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879651.322840:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879651.322842:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879651.322845:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879651.322846:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879651.322849:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879651.322851:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322853:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879651.322854:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879651.322856:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879651.322859:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879651.322860:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879651.322861:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879651.322863:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879651.322865:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879651.322867:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879651.322869:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322870:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879651.322872:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879651.322874:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879651.322876:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879651.322878:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879651.322879:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879651.322881:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879651.322883:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879651.322884:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879651.322886:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322888:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.322892:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879651.322894:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.322897:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879651.322899:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879651.385029:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879651.385041:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879651.385044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879651.385052:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879651.385058:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879651.385062:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.385067:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879651.385071:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879651.385074:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879652.089116:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879652.089125:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879652.089132:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879652.089134:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879652.089159:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879652.089166:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879652.089172:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879652.408972:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879652.408979:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879652.408982:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879652.408987:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879652.408990:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879652.408994:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879652.408997:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879652.409000:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879652.409002:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879653.113034:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879653.113043:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879653.113047:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879653.113048:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879653.113144:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879653.113150:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879653.113155:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879653.432979:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879653.432986:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879653.432988:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879653.432992:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879653.432996:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879653.432999:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879653.433002:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879653.433004:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879653.433007:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879654.137041:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879654.137048:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879654.137052:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879654.137053:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879654.137077:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879654.137079:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879654.137081:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879654.457024:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879654.457030:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879654.457032:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879654.457037:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879654.457040:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879654.457043:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879654.457046:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879654.457048:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879654.457051:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879655.161128:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879655.161141:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879655.161152:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879655.161159:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879655.161163:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879655.161170:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879655.161187:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879655.161192:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879655.161197:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879655.161201:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879655.161297:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879655.161301:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879655.161316:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879655.481069:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879655.481090:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879655.481093:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879655.481099:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879655.481104:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879655.481109:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879655.481112:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879655.481115:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879655.481119:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879656.185037:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879656.185045:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879656.185050:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879656.185051:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879656.185163:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879656.185169:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879656.185174:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879656.441018:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879656.441029:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879656.441032:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879656.441036:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879656.441045:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879656.441055:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10595 00000100:00080000:3.0:1788879656.441070:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879656.441075:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879656.441079:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879656.441091:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879656.441094:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879656.441100:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879656.441104:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879656.441118:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879656.441121:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879656.441128:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879656.441132:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879656.441136:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879656.441140:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879656.441144:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879656.441146:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879656.441149:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879656.441161:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879656.441163:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879656.441167:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879656.441169:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879656.441176:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879656.441192:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879656.441206:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290185088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879656.441223:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879656.441230:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879656.441253:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879656.441257:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879656.441261:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879656.441263:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879656.441265:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879656.441269:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879656.441272:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10595 00000100:00080000:3.0:1788879656.441278:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879656.441280:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879656.441282:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879656.441287:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879656.441289:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879656.441292:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879656.441295:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879656.441298:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879656.441300:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879656.441304:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879656.441306:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879656.441308:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879656.441315:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879656.441318:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879656.441320:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879656.441323:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:3.0:1788879656.441325:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879656.441346:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00000001:3.0:1788879656.441354:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879656.441359:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.441362:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879656.441367:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788879656.441367:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879656.441372:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879656.441373:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879656.441386:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879656.441389:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879656.441390:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879656.441400:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879656.441401:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879656.441415:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879656.441423:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879656.441427:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290185088:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879656.441435:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879656.441441:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1788879656.441441:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290185216/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879656.441447:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879656.441458:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879656.441471:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879656.441472:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879656.441480:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879656.441490:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879656.441494:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879656.441496:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879656.441500:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00080000:3.0:1788879656.441502:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879656.441510:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879656.441511:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879656.441516:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879656.441521:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879656.441526:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10595 00000100:00080000:3.0:1788879656.441531:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879656.441533:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879656.441534:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879656.441539:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879656.441540:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879656.441543:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879656.441545:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879656.441548:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879656.441551:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879656.441554:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879656.441558:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879656.441561:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:1.0:1788879656.441567:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879656.441568:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879656.441578:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.441590:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879656.441593:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879656.441599:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00100000:3.0:1788879656.441600:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879656.441607:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788879656.441609:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.441612:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879656.441619:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879656.441623:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.441627:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305268265 02000000:00000001:3.0:1788879656.441637:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879656.441642:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.441643:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879656.441646:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879656.441650:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879656.441653:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.441656:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305268265 00000100:00000001:3.0:1788879656.441664:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.441666:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879656.441675:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788879656.441678:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290185344/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879656.441679:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.441693:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879656.441710:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879656/real 1788879656] req@ffff94e50449ca80 x1875765290185088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879656.441739:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879656.441751:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879656.441751:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879656.441764:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.441765:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879656.441777:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879656.441778:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.441783:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.441787:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879656.441789:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879656.441808:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290185088/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879656.441809:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879656.441830:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879656.441841:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879656.441847:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879656.441852:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879656.441856:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879656.441859:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879656.441864:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10595 00000100:00000001:1.0:1788879656.441871:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.441875:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879656.441881:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788879656.441882:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879656.441904:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879656.441908:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879656.441915:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879656.441918:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879656.441920:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879656.441923:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879656.441925:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879656.441928:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879656.441934:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879656.441938:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879656.441941:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879656.441946:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879656.441952:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879656.441956:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879656.441960:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290185088:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879656.441977:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.441997:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879656.442000:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879656.442004:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788879656.442007:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879656.442011:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879656.442015:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:3.0:1788879656.442017:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1788879656.442022:0:2361:0:(genops.c:1216:class_import_put()) Process entered 02000000:00000001:3.0:1788879656.442022:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879656.442031:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000020:00000001:1.0:1788879656.442032:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000008:00100000:3.0:1788879656.442035:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879656.442041:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879656.442044:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1788879656.442045:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879656.442051:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879656.442073:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879656.442077:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879656.442077:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879656.442084:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442085:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879656.442100:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879656.442104:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1788879656.442105:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879656.442109:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879656.442113:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442115:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879656.442130:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879656.442131:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879656.442134:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879656.442140:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879656.442144:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879656.442147:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879656.442162:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879656.442183:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879656.442200:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290185216:192.168.203.129@tcp:38:kworker.0 00000100:00100000:3.0:1788879656.442204:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290185472/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879656.442209:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879656.442215:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879656.442221:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879656.442231:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879656.442247:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442266:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879656.442271:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:3.0:1788879656.442271:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879656.442281:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442283:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879656.442293:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879656.442314:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879656.442326:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879656.442376:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879656.442379:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442381:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879656.442384:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442401:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305267724 00000100:00000001:1.0:1788879656.442403:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305268265 00000100:00000001:1.0:1788879656.442406:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305267724 00000100:00000001:1.0:1788879656.442408:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305268265 00000100:00000001:1.0:1788879656.442410:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879656.442411:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442413:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879656.442417:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879656/real 1788879656] req@ffff94e50449f100 x1875765290185216/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879656.442446:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879656.442448:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879656.442452:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879656.442454:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879656.442456:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442458:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879656.442460:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879656.442465:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290185216/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879656.442474:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442477:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879656.442480:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442482:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879656.442483:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442486:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879656.442502:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879656.442505:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879656.442509:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879656.442512:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879656.442517:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290185216:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879656.442523:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879656.442525:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879656.442528:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879656.442533:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879656.442538:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879656.442542:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879656.442548:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290185344:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879656.442554:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879656.442559:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879656.442562:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879656.442566:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879656.442572:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442584:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879656.442587:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:1.0:1788879656.442594:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879656.442618:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879656.442626:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879656.442631:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879656.442634:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442637:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879656.442642:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442649:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305267722 00000100:00000001:1.0:1788879656.442654:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305268265 00000100:00000001:1.0:1788879656.442658:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305267722 00000100:00000001:1.0:1788879656.442662:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305268265 00000100:00000001:1.0:1788879656.442665:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879656.442669:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442672:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879656.442680:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879656/real 1788879656] req@ffff94e50449c000 x1875765290185344/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879656.442718:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879656.442720:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879656.442722:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879656.442724:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879656.442726:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442728:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879656.442730:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879656.442735:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290185344/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879656.442742:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442745:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879656.442748:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442750:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879656.442751:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442753:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879656.442767:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879656.442771:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879656.442775:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879656.442778:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879656.442782:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290185344:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879656.442787:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442788:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879656.442790:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879656.442791:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879656.442794:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879656.442795:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879656.442796:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879656.442799:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879656.442800:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879656.442802:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879656.442803:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442805:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879656.442806:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879656.442807:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879656.442809:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879656.442810:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879656.442811:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879656.442812:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879656.442813:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879656.442814:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879656.442815:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442816:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442818:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879656.442820:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442821:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879656.442822:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879656.442823:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879656.442825:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879656.442826:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879656.442828:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879656.442830:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879656.442831:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879656.442834:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290185472:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879656.442837:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879656.442839:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879656.442840:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879656.442842:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879656.442844:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442849:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879656.442850:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:1.0:1788879656.442854:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879656.442865:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879656.442868:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879656.442870:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879656.442871:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442872:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879656.442874:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442878:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305267722 00000100:00000001:1.0:1788879656.442880:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305268265 00000100:00000001:1.0:1788879656.442882:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305267722 00000100:00000001:1.0:1788879656.442884:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305268265 00000100:00000001:1.0:1788879656.442886:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879656.442887:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442889:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879656.442892:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879656/real 1788879656] req@ffff94e50449e300 x1875765290185472/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879656.442897:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879656.442898:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879656.442900:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879656.442901:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879656.442903:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442905:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879656.442906:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879656.442909:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290185472/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879656.442914:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442915:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879656.442917:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442919:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879656.442920:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442921:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879656.442932:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879656.442957:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879656.442959:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879656.442961:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879656.442964:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290185472:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879656.442968:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442970:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879656.442972:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879656.442974:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879656.442976:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879656.442978:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879656.442979:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879656.442982:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879656.442983:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879656.442986:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879656.442987:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442989:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.442993:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879656.442995:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.442997:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879656.442999:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879656.505030:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879656.505038:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879656.505040:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879656.505045:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879656.505048:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879656.505052:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.505054:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879656.505057:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879656.505059:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879657.209059:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879657.209070:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879657.209073:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879657.209076:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879657.209081:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879657.209083:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879657.209085:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879657.209088:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879657.209090:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879657.209091:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879657.209221:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879657.209230:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879657.209236:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879657.529030:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879657.529057:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879657.529060:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879657.529067:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879657.529072:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879657.529077:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879657.529080:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879657.529084:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879657.529099:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879658.233079:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879658.233088:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879658.233093:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879658.233094:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879658.233119:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879658.233121:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879658.233124:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879658.552996:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879658.553005:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879658.553008:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879658.553014:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879658.553019:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879658.553024:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879658.553027:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879658.553031:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879658.553034:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879659.257027:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879659.257035:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879659.257040:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879659.257041:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879659.257086:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879659.257088:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879659.257092:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879659.576959:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879659.576964:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879659.576966:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879659.576970:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879659.576972:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879659.576976:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879659.576978:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879659.576981:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879659.576983:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879660.280986:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879660.280991:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879660.280995:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879660.280996:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879660.281046:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879660.281049:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879660.281052:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879660.600963:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879660.600969:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879660.600971:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879660.600975:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879660.600978:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879660.600981:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879660.600983:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879660.600985:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879660.600988:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879661.304991:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879661.304996:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879661.305001:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879661.305001:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879661.305048:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879661.305052:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879661.305055:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879661.560988:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879661.560996:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879661.560999:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879661.561002:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879661.561006:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879661.561013:0:80852:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10600 00000100:00080000:1.0:1788879661.561023:0:80852:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879661.561026:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879661.561027:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879661.561035:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879661.561037:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879661.561041:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879661.561043:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879661.561045:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879661.561047:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879661.561051:0:80852:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879661.561054:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879661.561057:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879661.561060:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879661.561062:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879661.561063:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879661.561065:0:80852:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879661.561073:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879661.561074:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879661.561077:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879661.561079:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879661.561082:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879661.561093:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879661.561101:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009f80 x1875765290185600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879661.561110:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879661.561113:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879661.561125:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879661.561128:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879661.561131:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879661.561132:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879661.561134:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879661.561135:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879661.561138:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879661.561140:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1788879661.561143:0:80852:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10600 00000100:00000001:2.0:1788879661.561144:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879661.561148:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:1.0:1788879661.561150:0:80852:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879661.561151:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879661.561155:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1788879661.561158:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788879661.561159:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879661.561161:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1788879661.561164:0:80852:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1788879661.561165:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879661.561167:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879661.561171:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00100000:2.0:1788879661.561172:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290185600:192.168.203.129@tcp:250:kworker.0 00000020:00000001:1.0:1788879661.561174:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879661.561178:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788879661.561179:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879661.561182:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879661.561183:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1788879661.561187:0:80852:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:2.0:1788879661.561188:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879661.561193:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879661.561193:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879661.561196:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879661.561197:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879661.561201:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879661.561204:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879661.561206:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879661.561209:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879661.561212:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561213:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879661.561214:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561216:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286516096 : -117763423035520 : ffff94e511009f80) 02000000:00000001:1.0:1788879661.561220:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879661.561221:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1788879661.561224:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879661.561227:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879661.561228:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879661.561230:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879661.561235:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879661.561239:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008380 x1875765290185728/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879661.561245:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879661.561247:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879661.561249:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879661.561251:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879661.561254:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879661.561257:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000800:00000100:2.0:1788879661.561258:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:1.0:1788879661.561259:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879661.561263:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879661.561267:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10600 00000100:00000001:2.0:1788879661.561268:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:1.0:1788879661.561270:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879661.561272:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879661.561273:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879661.561274:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1788879661.561277:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879661.561278:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879661.561280:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879661.561281:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:1.0:1788879661.561283:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879661.561286:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879661.561287:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879661.561290:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879661.561291:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879661.561293:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879661.561294:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879661.561295:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879661.561312:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305273385 02000000:00000001:1.0:1788879661.561313:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879661.561317:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879661.561318:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879661.561320:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879661.561322:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00000001:1.0:1788879661.561323:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879661.561327:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561328:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305273385 00000100:00000001:2.0:1788879661.561333:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1788879661.561337:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879661.561338:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879661.561341:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561342:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879661.561344:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879661.561346:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1788879661.561347:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879661.561350:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879661.561354:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879661/real 1788879661] req@ffff94e511009f80 x1875765290185600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879661.561357:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879661.561367:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009180 x1875765290185856/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1788879661.561369:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879661.561372:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879661.561377:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879661.561378:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879661.561380:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561382:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879661.561384:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561385:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788879661.561387:0:80852:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879661.561388:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879661.561392:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788879661.561393:0:80852:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879661.561396:0:80852:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879661.561399:0:80852:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1788879661.561400:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009f80 x1875765290185600/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879661.561401:0:80852:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879661.561407:0:80852:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10600 00000100:00000001:2.0:1788879661.561414:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788879661.561415:0:80852:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879661.561418:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879661.561418:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879661.561423:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879661.561424:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879661.561426:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879661.561429:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1788879661.561431:0:80852:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879661.561432:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000020:00000001:1.0:1788879661.561437:0:80852:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879661.561437:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879661.561442:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:1.0:1788879661.561443:0:80852:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00080000:2.0:1788879661.561446:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000020:00000001:1.0:1788879661.561451:0:80852:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879661.561451:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879661.561458:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290185600:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879661.561459:0:80852:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879661.561464:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879661.561465:0:80852:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879661.561466:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879661.561470:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879661.561472:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:1.0:1788879661.561474:0:80852:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:2.0:1788879661.561475:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879661.561478:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:1.0:1788879661.561483:0:80852:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879661.561483:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879661.561486:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879661.561488:0:80852:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879661.561489:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879661.561494:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 02000000:00000001:1.0:1788879661.561495:0:80852:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879661.561496:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788879661.561500:0:80852:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561502:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879661.561506:0:80852:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879661.561510:0:80852:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879661.561512:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000008:00000001:1.0:1788879661.561514:0:80852:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561516:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879661.561518:0:80852:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561521:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1788879661.561525:0:80852:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879661.561526:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1788879661.561529:0:80852:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561530:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879661.561532:0:80852:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879661.561534:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788879661.561535:0:80852:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879661.561537:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879661.561539:0:80852:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879661.561543:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788879661.561546:0:80852:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879661.561547:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879661.561551:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879661.561554:0:80852:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009880 x1875765290185984/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788879661.561557:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290185728:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879661.561566:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879661.561569:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1788879661.561570:0:80852:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788879661.561572:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879661.561575:0:80852:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879661.561576:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788879661.561580:0:80852:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879661.561581:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561591:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879661.561593:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508928 : -117763423042688 : ffff94e511008380) 00000100:00000001:2.0:1788879661.561597:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879661.561613:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879661.561619:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879661.561621:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879661.561623:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561624:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879661.561626:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561629:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305272843 00000100:00000001:2.0:1788879661.561631:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305273385 00000100:00000001:2.0:1788879661.561633:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305272843 00000100:00000001:2.0:1788879661.561634:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305273385 00000100:00000001:2.0:1788879661.561636:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879661.561638:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561639:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879661.561642:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879661/real 1788879661] req@ffff94e511008380 x1875765290185728/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879661.561649:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879661.561650:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879661.561651:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879661.561652:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879661.561654:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561655:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879661.561657:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879661.561659:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008380 x1875765290185728/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879661.561664:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561666:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879661.561668:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561669:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879661.561671:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561672:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879661.561695:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879661.561698:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879661.561702:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879661.561704:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879661.561706:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290185728:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879661.561709:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879661.561710:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879661.561712:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879661.561713:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879661.561714:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879661.561716:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879661.561717:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290185856:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879661.561719:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879661.561720:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879661.561722:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879661.561723:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879661.561725:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561728:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879661.561729:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286512512 : -117763423039104 : ffff94e511009180) 00000100:00000001:2.0:1788879661.561732:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879661.561739:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879661.561742:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879661.561743:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879661.561744:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561745:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879661.561747:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561749:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305272841 00000100:00000001:2.0:1788879661.561751:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305273385 00000100:00000001:2.0:1788879661.561752:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305272841 00000100:00000001:2.0:1788879661.561754:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305273385 00000100:00000001:2.0:1788879661.561756:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879661.561757:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561758:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879661.561761:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879661/real 1788879661] req@ffff94e511009180 x1875765290185856/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879661.561766:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879661.561767:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879661.561768:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879661.561769:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879661.561770:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561771:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879661.561772:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879661.561774:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009180 x1875765290185856/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879661.561779:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561780:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879661.561782:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561783:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879661.561784:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561785:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879661.561796:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879661.561797:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879661.561799:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879661.561800:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879661.561802:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290185856:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879661.561804:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561806:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879661.561807:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879661.561808:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879661.561809:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879661.561810:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879661.561811:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879661.561813:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879661.561814:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879661.561815:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879661.561816:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561817:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879661.561818:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879661.561819:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879661.561820:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879661.561821:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879661.561822:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879661.561823:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879661.561823:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879661.561824:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879661.561825:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561826:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561828:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879661.561829:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561831:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879661.561831:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879661.561832:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879661.561833:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879661.561834:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879661.561835:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879661.561836:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879661.561837:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879661.561840:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290185984:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879661.561842:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879661.561843:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879661.561844:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879661.561845:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879661.561846:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561849:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879661.561850:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286514304 : -117763423037312 : ffff94e511009880) 00000100:00000001:2.0:1788879661.561852:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879661.561860:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879661.561864:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879661.561866:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879661.561868:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561870:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879661.561872:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561876:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305272841 00000100:00000001:2.0:1788879661.561878:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305273385 00000100:00000001:2.0:1788879661.561880:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305272841 00000100:00000001:2.0:1788879661.561883:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305273385 00000100:00000001:2.0:1788879661.561885:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879661.561887:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.561889:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879661.561893:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879661/real 1788879661] req@ffff94e511009880 x1875765290185984/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879661.561931:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879661.561933:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879661.561937:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879661.561939:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879661.561941:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561943:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879661.561944:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879661.561948:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009880 x1875765290185984/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879661.561953:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561955:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879661.561957:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561959:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879661.561960:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561961:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879661.561975:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879661.561976:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879661.561978:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879661.561980:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879661.561982:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290185984:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879661.561985:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561986:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879661.561987:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879661.561988:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879661.561990:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879661.561991:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879661.561992:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879661.561994:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879661.561994:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879661.561996:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879661.561997:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.561998:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879661.562000:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879661.562001:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879661.562002:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879661.562004:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879661.624975:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879661.624980:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879661.624982:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879661.624986:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879661.624989:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879661.624993:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879661.624995:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879661.624997:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879661.624998:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879662.329051:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879662.329061:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879662.329072:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879662.329074:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879662.329123:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879662.329129:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879662.329134:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879662.649037:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879662.649046:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879662.649049:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879662.649056:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879662.649061:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879662.649066:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879662.649070:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879662.649074:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879662.649077:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879663.353035:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879663.353044:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879663.353051:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879663.353054:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879663.353089:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879663.353093:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879663.353100:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879663.673001:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879663.673011:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879663.673016:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879663.673042:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879663.673049:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879663.673056:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879663.673064:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879663.673068:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879663.673073:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879664.377007:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879664.377015:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879664.377023:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879664.377025:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879664.377106:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879664.377110:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879664.377114:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879664.697003:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879664.697012:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879664.697017:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879664.697024:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879664.697029:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879664.697034:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879664.697047:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879664.697052:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879664.697056:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879665.401011:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879665.401019:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879665.401025:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879665.401029:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879665.401032:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879665.401036:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879665.401040:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879665.401043:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879665.401045:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879665.401048:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879665.401154:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879665.401162:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879665.401169:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879665.721012:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879665.721020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879665.721023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879665.721029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879665.721034:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879665.721039:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879665.721041:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879665.721044:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879665.721047:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879666.424981:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879666.424989:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879666.424994:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879666.424995:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879666.425017:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879666.425019:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879666.425021:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879666.681027:0:178259:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879666.681041:0:178259:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879666.681044:0:178259:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879666.681049:0:178259:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879666.681056:0:178259:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879666.681066:0:178259:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10605 00000100:00080000:1.0:1788879666.681079:0:178259:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879666.681083:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879666.681086:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879666.681114:0:178259:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879666.681117:0:178259:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879666.681121:0:178259:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879666.681124:0:178259:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879666.681126:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879666.681128:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879666.681132:0:178259:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879666.681136:0:178259:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879666.681139:0:178259:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879666.681143:0:178259:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879666.681147:0:178259:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879666.681150:0:178259:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879666.681153:0:178259:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879666.681181:0:178259:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879666.681185:0:178259:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879666.681210:0:178259:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879666.681219:0:178259:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879666.681227:0:178259:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879666.681246:0:178259:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879666.681259:0:178259:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100a680 x1875765290186112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879666.681307:0:178259:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879666.681311:0:178259:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879666.681322:0:178259:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879666.681325:0:178259:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879666.681327:0:178259:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879666.681329:0:178259:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879666.681330:0:178259:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879666.681332:0:178259:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879666.681334:0:178259:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10605 00000100:00080000:1.0:1788879666.681338:0:178259:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879666.681339:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879666.681341:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879666.681344:0:178259:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879666.681346:0:178259:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879666.681347:0:178259:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879666.681349:0:178259:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879666.681349:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879666.681354:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879666.681355:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879666.681355:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879666.681358:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879666.681360:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879666.681362:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879666.681366:0:178259:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:2.0:1788879666.681368:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788879666.681369:0:178259:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879666.681372:0:178259:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879666.681372:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879666.681377:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879666.681379:0:178259:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879666.681383:0:178259:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879666.681383:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290186112:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879666.681387:0:178259:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879666.681392:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:1.0:1788879666.681393:0:178259:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788879666.681396:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:1.0:1788879666.681397:0:178259:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879666.681400:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879666.681401:0:178259:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879666.681406:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879666.681408:0:178259:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879666.681411:0:178259:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879666.681411:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879666.681413:0:178259:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879666.681416:0:178259:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879666.681418:0:178259:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879666.681423:0:178259:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879666.681429:0:178259:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b800 x1875765290186240/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879666.681429:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879666.681432:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286517888 : -117763423033728 : ffff94e51100a680) 00000100:00000001:2.0:1788879666.681438:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879666.681442:0:178259:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879666.681444:0:178259:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879666.681447:0:178259:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879666.681449:0:178259:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879666.681451:0:178259:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879666.681453:0:178259:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879666.681454:0:178259:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879666.681457:0:178259:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879666.681459:0:178259:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10605 00000100:00080000:1.0:1788879666.681468:0:178259:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879666.681470:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879666.681471:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879666.681475:0:178259:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000100:2.0:1788879666.681475:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:1.0:1788879666.681478:0:178259:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879666.681481:0:178259:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879666.681484:0:178259:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879666.681486:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879666.681487:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879666.681489:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:1.0:1788879666.681492:0:178259:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879666.681495:0:178259:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.681495:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1788879666.681498:0:178259:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879666.681499:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879666.681502:0:178259:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879666.681503:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:1.0:1788879666.681506:0:178259:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879666.681509:0:178259:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879666.681510:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1788879666.681511:0:178259:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879666.681517:0:178259:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879666.681519:0:178259:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.681519:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305278505 02000000:00000001:1.0:1788879666.681523:0:178259:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879666.681525:0:178259:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879666.681527:0:178259:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879666.681527:0:178259:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879666.681529:0:178259:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.681533:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788879666.681534:0:178259:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879666.681541:0:178259:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b480 x1875765290186368/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879666.681547:0:178259:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879666.681548:0:178259:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879666.681550:0:178259:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879666.681552:0:178259:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879666.681554:0:178259:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879666.681555:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305278505 00000100:00000001:1.0:1788879666.681556:0:178259:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879666.681558:0:178259:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879666.681582:0:178259:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879666.681586:0:178259:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10605 00000100:00000001:2.0:1788879666.681590:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1788879666.681592:0:178259:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879666.681599:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879666.681603:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879666.681604:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879666.681616:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879666.681617:0:178259:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879666.681623:0:178259:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879666.681626:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:1.0:1788879666.681629:0:178259:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:1.0:1788879666.681635:0:178259:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879666.681642:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1788879666.681643:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879666/real 1788879666] req@ffff94e51100a680 x1875765290186112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879666.681645:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879666.681660:0:178259:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000400:00000001:2.0:1788879666.681670:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879666.681676:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879666.681679:0:178259:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.681685:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:1.0:1788879666.681686:0:178259:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879666.681694:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:1.0:1788879666.681696:0:178259:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879666.681700:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788879666.681702:0:178259:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.681706:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879666.681708:0:178259:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879666.681712:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:1.0:1788879666.681714:0:178259:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879666.681720:0:178259:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879666.681725:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100a680 x1875765290186112/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879666.681726:0:178259:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879666.681739:0:178259:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879666.681743:0:178259:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.681748:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879666.681749:0:178259:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879666.681752:0:178259:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879666.681754:0:178259:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879666.681760:0:178259:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879666.681770:0:178259:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100aa00 x1875765290186496/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879666.681772:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879666.681783:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.681790:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879666.681794:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879666.681804:0:178259:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879666.681805:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879666.681813:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879666.681814:0:178259:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879666.681823:0:178259:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879666.681825:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879666.681836:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879666.681840:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879666.681846:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290186112:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879666.681855:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.681859:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879666.681862:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879666.681865:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879666.681870:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879666.681873:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879666.681875:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879666.681880:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879666.681882:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879666.681886:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879666.681887:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.681890:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.681898:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879666.681901:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.681934:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879666.681936:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879666.681938:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879666.681940:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879666.681942:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879666.681945:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879666.681948:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879666.681951:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879666.681955:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290186240:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879666.681959:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879666.681962:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879666.681964:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879666.681967:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879666.681971:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.681991:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879666.681994:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:2.0:1788879666.682005:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879666.682026:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879666.682033:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879666.682037:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879666.682039:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.682041:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879666.682044:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.682050:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305277964 00000100:00000001:2.0:1788879666.682054:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305278505 00000100:00000001:2.0:1788879666.682056:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305277964 00000100:00000001:2.0:1788879666.682059:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305278505 00000100:00000001:2.0:1788879666.682062:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879666.682064:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.682067:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879666.682072:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879666/real 1788879666] req@ffff94e51100b800 x1875765290186240/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879666.682082:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879666.682084:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879666.682087:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879666.682089:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879666.682091:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682093:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879666.682095:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879666.682116:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290186240/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879666.682121:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682122:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879666.682124:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682125:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879666.682126:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682128:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879666.682144:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879666.682147:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879666.682151:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879666.682154:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879666.682158:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290186240:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879666.682162:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879666.682164:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879666.682182:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879666.682186:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879666.682190:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879666.682194:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879666.682211:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290186368:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879666.682217:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879666.682221:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879666.682223:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879666.682226:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879666.682230:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.682239:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879666.682241:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:2.0:1788879666.682246:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879666.682304:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879666.682308:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879666.682310:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879666.682311:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.682312:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879666.682314:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.682317:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305277962 00000100:00000001:2.0:1788879666.682319:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305278505 00000100:00000001:2.0:1788879666.682321:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305277962 00000100:00000001:2.0:1788879666.682322:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305278505 00000100:00000001:2.0:1788879666.682323:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879666.682325:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.682327:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879666.682329:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879666/real 1788879666] req@ffff94e51100b480 x1875765290186368/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879666.682335:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879666.682336:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879666.682338:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879666.682339:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879666.682340:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682341:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879666.682342:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879666.682345:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290186368/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879666.682349:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682350:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879666.682352:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682353:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879666.682354:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682355:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879666.682367:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879666.682369:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879666.682371:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879666.682372:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879666.682374:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290186368:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879666.682377:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879666.682378:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879666.682379:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879666.682380:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879666.682381:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879666.682382:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879666.682384:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290186496:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879666.682388:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879666.682389:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879666.682390:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879666.682391:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879666.682394:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.682397:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879666.682398:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:2.0:1788879666.682400:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879666.682408:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879666.682410:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879666.682412:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879666.682413:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.682414:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879666.682415:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.682418:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305277962 00000100:00000001:2.0:1788879666.682419:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305278505 00000100:00000001:2.0:1788879666.682421:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305277962 00000100:00000001:2.0:1788879666.682422:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305278505 00000100:00000001:2.0:1788879666.682424:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879666.682425:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.682426:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879666.682428:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879666/real 1788879666] req@ffff94e51100aa00 x1875765290186496/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879666.682433:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879666.682434:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879666.682435:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879666.682436:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879666.682438:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682439:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879666.682440:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879666.682442:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290186496/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879666.682451:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682452:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879666.682454:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682455:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879666.682456:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682457:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879666.682469:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879666.682470:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879666.682472:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879666.682473:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879666.682475:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290186496:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879666.682478:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682479:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879666.682480:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879666.682481:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879666.682483:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879666.682484:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879666.682484:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879666.682486:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879666.682487:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879666.682489:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879666.682490:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682491:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879666.682491:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879666.682492:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879666.682493:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879666.682494:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879666.682495:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879666.682496:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879666.682497:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879666.682497:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879666.682498:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682499:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879666.682500:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879666.682501:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879666.682502:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879666.682503:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879666.682504:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879666.682505:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879666.682506:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879666.682506:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879666.682507:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682509:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879666.682512:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879666.682514:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879666.682516:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879666.682517:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879666.745023:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879666.745031:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879666.745033:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879666.745040:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879666.745044:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879666.745049:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879666.745053:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879666.745057:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879666.745060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879667.448997:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879667.449005:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879667.449007:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879667.449011:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879667.449017:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879667.449019:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879667.449021:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879667.449024:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879667.449028:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879667.449029:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879667.449049:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879667.449052:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879667.449055:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879667.769066:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879667.769075:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879667.769078:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879667.769084:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879667.769088:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879667.769094:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879667.769097:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879667.769100:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879667.769104:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879668.473036:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879668.473046:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879668.473058:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879668.473062:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879668.473152:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879668.473158:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879668.473163:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879668.793047:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879668.793057:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879668.793061:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879668.793070:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879668.793076:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879668.793083:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879668.793088:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879668.793093:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879668.793099:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879669.497038:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879669.497046:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879669.497053:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879669.497055:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879669.497132:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879669.497137:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879669.497143:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879669.816996:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879669.817004:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879669.817008:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879669.817014:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879669.817019:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879669.817024:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879669.817027:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879669.817030:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879669.817034:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879670.521025:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879670.521033:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879670.521040:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879670.521042:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879670.521067:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879670.521070:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879670.521074:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879670.841124:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879670.841136:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879670.841139:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879670.841146:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879670.841151:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879670.841156:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879670.841159:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879670.841163:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879670.841167:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879671.545036:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879671.545046:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879671.545056:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879671.545059:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879671.545130:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879671.545136:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879671.545142:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879671.801131:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879671.801144:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879671.801147:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879671.801151:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879671.801157:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879671.801166:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10610 00000100:00080000:3.0:1788879671.801182:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879671.801185:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879671.801187:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879671.801197:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879671.801201:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879671.801205:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879671.801208:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879671.801210:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879671.801212:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879671.801216:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879671.801219:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879671.801222:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879671.801245:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879671.801249:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879671.801252:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879671.801255:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879671.801274:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879671.801277:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879671.801283:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879671.801287:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879671.801294:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879671.801315:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879671.801331:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290186624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879671.801344:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879671.801349:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879671.801374:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879671.801378:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879671.801382:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879671.801384:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879671.801386:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879671.801389:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879671.801392:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10610 00000100:00080000:3.0:1788879671.801396:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879671.801399:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879671.801400:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879671.801405:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879671.801406:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879671.801409:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879671.801412:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879671.801414:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879671.801416:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879671.801422:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879671.801426:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879671.801429:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879671.801440:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879671.801446:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.801448:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879671.801452:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879671.801463:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00100000:3.0:1788879671.801465:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879671.801472:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00000001:3.0:1788879671.801476:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.801481:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879671.801485:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.801489:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879671.801497:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879671.801500:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879671.801506:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879671.801506:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879671.801509:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879671.801521:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879671.801522:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879671.801529:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879671.801533:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879671.801540:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290186752/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788879671.801553:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290186624:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879671.801559:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879671.801570:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.801572:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879671.801578:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879671.801581:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1788879671.801585:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1788879671.801587:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879671.801601:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:2.0:1788879671.801602:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879671.801613:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879671.801616:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1788879671.801620:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879671.801621:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879671.801629:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10610 00000100:00080000:3.0:1788879671.801636:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879671.801640:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879671.801643:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879671.801646:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879671.801648:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1788879671.801649:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879671.801655:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000020:00000001:3.0:1788879671.801656:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879671.801672:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879671.801674:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879671.801686:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879671.801690:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879671.801695:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879671.801698:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879671.801701:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879671.801705:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879671.801707:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879671.801709:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879671.801711:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879671.801714:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879671.801717:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879671.801722:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879671.801723:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879671.801725:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879671.801727:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879671.801731:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879671.801739:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788879671.801741:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00100000:3.0:1788879671.801751:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290186880/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879671.801760:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879671.801761:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879671.801765:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.801770:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879671.801771:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.801780:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879671.801783:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879671.801790:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:3.0:1788879671.801797:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879671.801802:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879671.801803:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879671.801806:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879671.801811:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305283625 00000100:00000001:3.0:1788879671.801812:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879671.801825:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10610 00000100:00080000:3.0:1788879671.801831:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879671.801833:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879671.801833:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879671.801838:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879671.801846:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879671.801847:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879671.801849:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879671.801851:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879671.801853:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879671.801855:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879671.801860:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879671.801863:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305283625 00000100:00000001:3.0:1788879671.801865:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879671.801873:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879671.801878:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1788879671.801879:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879671.801890:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.801893:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879671.801894:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879671.801901:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879671.801968:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.801976:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000008:00000001:3.0:1788879671.801988:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879671.801993:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879671.801996:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879671/real 1788879671] req@ffff94e50449c700 x1875765290186624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879671.801999:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879671.802002:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879671.802008:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879671.802012:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879671.802018:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879671.802034:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879671.802051:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290187008/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1788879671.802052:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879671.802059:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879671.802067:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879671.802081:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879671.802082:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879671.802086:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802093:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879671.802093:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.802098:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879671.802107:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879671.802111:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290186624/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879671.802124:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802129:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879671.802136:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802138:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879671.802140:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802144:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879671.802149:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879671.802155:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879671.802162:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879671.802171:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879671.802192:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290186624:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879671.802199:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802202:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879671.802205:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879671.802208:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879671.802213:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879671.802216:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879671.802218:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879671.802223:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879671.802249:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879671.802256:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879671.802259:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802264:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802277:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879671.802281:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.802285:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879671.802289:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879671.802292:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879671.802295:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879671.802298:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879671.802304:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879671.802308:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879671.802313:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879671.802320:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290186752:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879671.802328:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879671.802333:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879671.802336:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879671.802340:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879671.802345:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.802355:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879671.802357:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:2.0:1788879671.802364:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879671.802386:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879671.802393:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879671.802396:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879671.802398:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.802401:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879671.802404:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.802410:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305283084 00000100:00000001:2.0:1788879671.802414:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305283625 00000100:00000001:2.0:1788879671.802417:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305283084 00000100:00000001:2.0:1788879671.802419:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305283625 00000100:00000001:2.0:1788879671.802422:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879671.802424:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.802427:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879671.802432:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879671/real 1788879671] req@ffff94e50449f480 x1875765290186752/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879671.802446:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879671.802448:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879671.802453:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879671.802456:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879671.802459:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802462:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879671.802465:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879671.802473:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290186752/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879671.802487:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802492:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879671.802496:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802500:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879671.802502:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802506:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879671.802532:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879671.802538:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879671.802543:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879671.802546:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879671.802551:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290186752:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879671.802557:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879671.802559:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879671.802561:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879671.802564:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879671.802567:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879671.802569:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879671.802573:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290186880:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879671.802577:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879671.802580:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879671.802582:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879671.802584:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879671.802588:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.802597:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879671.802599:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:2.0:1788879671.802604:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879671.802620:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879671.802627:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879671.802630:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879671.802632:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.802634:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879671.802637:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.802642:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305283082 00000100:00000001:2.0:1788879671.802645:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305283625 00000100:00000001:2.0:1788879671.802648:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305283082 00000100:00000001:2.0:1788879671.802650:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305283625 00000100:00000001:2.0:1788879671.802653:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879671.802655:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.802657:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879671.802662:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879671/real 1788879671] req@ffff94e50449f800 x1875765290186880/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879671.802681:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879671.802682:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879671.802686:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879671.802688:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879671.802689:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802691:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879671.802693:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879671.802698:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290186880/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879671.802706:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802709:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879671.802711:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802713:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879671.802715:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802717:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879671.802734:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879671.802737:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879671.802740:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879671.802743:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879671.802746:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290186880:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879671.802750:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879671.802752:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879671.802754:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879671.802757:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879671.802759:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879671.802761:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879671.802765:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290187008:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879671.802768:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879671.802770:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879671.802773:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879671.802775:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879671.802780:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.802785:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879671.802787:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:2.0:1788879671.802791:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879671.802802:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879671.802806:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879671.802809:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879671.802810:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.802812:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879671.802815:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.802819:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305283082 00000100:00000001:2.0:1788879671.802822:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305283625 00000100:00000001:2.0:1788879671.802825:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305283082 00000100:00000001:2.0:1788879671.802828:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305283625 00000100:00000001:2.0:1788879671.802831:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879671.802833:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.802835:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879671.802839:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879671/real 1788879671] req@ffff94e50449c380 x1875765290187008/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879671.802848:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879671.802849:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879671.802852:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879671.802855:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879671.802857:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802860:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879671.802863:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879671.802870:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290187008/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879671.802883:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802887:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879671.802892:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802896:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879671.802898:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.802943:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879671.802966:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879671.802973:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879671.802981:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879671.802985:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879671.802992:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290187008:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879671.802999:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.803003:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879671.803006:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879671.803008:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879671.803014:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879671.803016:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879671.803019:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879671.803024:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879671.803027:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879671.803031:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879671.803034:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.803037:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879671.803039:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879671.803042:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879671.803046:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879671.803048:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879671.803050:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879671.803054:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879671.803056:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879671.803059:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879671.803060:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.803063:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879671.803064:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879671.803066:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879671.803068:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879671.803069:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879671.803070:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879671.803072:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879671.803074:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879671.803076:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879671.803078:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.803080:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879671.803086:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879671.803088:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879671.803090:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879671.803092:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879671.865047:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879671.865058:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879671.865062:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879671.865068:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879671.865073:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879671.865077:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879671.865080:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879671.865084:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879671.865088:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879672.568980:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879672.568985:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879672.568990:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879672.568991:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879672.569049:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879672.569052:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879672.569055:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879672.888979:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879672.888985:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879672.888988:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879672.888992:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879672.888994:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879672.888998:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879672.889000:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879672.889002:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879672.889004:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879673.593058:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879673.593064:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879673.593069:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879673.593071:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879673.593147:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879673.593150:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879673.593154:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879673.912956:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879673.912963:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879673.912964:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879673.912968:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879673.912971:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879673.912975:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879673.912977:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879673.912978:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879673.912980:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879674.616976:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879674.616981:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879674.616985:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879674.616986:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879674.617002:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879674.617004:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879674.617006:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879674.936957:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879674.936963:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879674.936965:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879674.936969:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879674.936972:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879674.936976:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879674.936978:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879674.936980:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879674.936982:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879675.640998:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879675.641004:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879675.641010:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879675.641013:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879675.641016:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879675.641020:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879675.641023:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879675.641025:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879675.641027:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879675.641029:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879675.641052:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879675.641054:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879675.641057:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879675.961010:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879675.961018:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879675.961021:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879675.961027:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879675.961032:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879675.961037:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879675.961041:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879675.961045:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879675.961049:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879676.665070:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879676.665078:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879676.665085:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879676.665087:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879676.665267:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879676.665274:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879676.665281:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879676.921050:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879676.921062:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879676.921065:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879676.921069:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879676.921086:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879676.921094:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10615 00000100:00080000:0.0:1788879676.921107:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879676.921110:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879676.921112:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879676.921131:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879676.921134:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879676.921138:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879676.921141:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879676.921144:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879676.921145:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879676.921151:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879676.921153:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879676.921157:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879676.921162:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879676.921165:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879676.921169:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879676.921171:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879676.921184:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879676.921187:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879676.921204:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879676.921208:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879676.921214:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879676.921234:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879676.921245:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290187136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879676.921257:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879676.921262:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879676.921279:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879676.921283:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879676.921287:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879676.921289:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879676.921291:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879676.921294:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879676.921297:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10615 00000100:00080000:0.0:1788879676.921301:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879676.921303:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879676.921311:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879676.921316:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879676.921318:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879676.921321:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879676.921325:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879676.921327:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879676.921329:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879676.921333:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879676.921336:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879676.921338:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879676.921340:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788879676.921346:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879676.921347:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879676.921353:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1788879676.921354:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879676.921360:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879676.921360:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879676.921366:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1788879676.921369:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879676.921375:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879676.921376:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1788879676.921378:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879676.921382:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879676.921386:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879676.921388:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879676.921391:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879676.921396:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879676.921399:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1788879676.921399:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290187136:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788879676.921405:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.921410:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1788879676.921415:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879676.921417:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879676.921423:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879676.921428:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1788879676.921430:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290187264/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788879676.921436:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879676.921446:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879676.921449:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879676.921454:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879676.921457:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879676.921461:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879676.921462:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1788879676.921464:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879676.921467:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00080000:0.0:1788879676.921469:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879676.921476:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879676.921477:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879676.921484:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10615 00000100:00080000:0.0:1788879676.921489:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879676.921491:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879676.921493:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879676.921497:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879676.921499:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879676.921502:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879676.921504:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879676.921507:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879676.921508:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879676.921522:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:2.0:1788879676.921528:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:0.0:1788879676.921530:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879676.921535:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879676.921539:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879676.921542:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879676.921545:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879676.921546:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00100000:0.0:1788879676.921548:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879676.921553:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.921555:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879676.921556:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.921562:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879676.921566:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879676.921567:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:0.0:1788879676.921572:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879676.921576:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879676.921576:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879676.921578:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879676.921584:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.921585:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305288745 00000100:00000001:0.0:1788879676.921591:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879676.921600:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290187392/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879676.921609:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879676.921611:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.921613:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788879676.921615:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879676.921622:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879676.921626:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305288745 00000100:00080000:0.0:1788879676.921627:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879676.921633:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879676.921637:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879676.921638:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1788879676.921643:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879676.921647:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1788879676.921651:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10615 00000100:00000001:2.0:1788879676.921653:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.921659:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:0.0:1788879676.921661:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879676.921665:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879676.921670:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:2.0:1788879676.921671:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879676/real 1788879676] req@ffff94e508979c00 x1875765290187136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1788879676.921674:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879676.921677:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879676.921680:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879676.921682:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879676.921689:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879676.921692:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879676.921703:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879676.921710:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879676.921712:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1788879676.921713:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:2.0:1788879676.921718:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1788879676.921721:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879676.921725:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.921726:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:0.0:1788879676.921728:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879676.921733:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000008:00100000:0.0:1788879676.921734:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879676.921739:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788879676.921740:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.921748:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879676.921749:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.921755:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879676.921758:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879676.921763:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879676.921765:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290187136/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879676.921767:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879676.921771:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879676.921775:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.921785:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879676.921786:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.921792:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879676.921798:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879676.921799:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290187520/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879676.921801:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879676.921806:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.921812:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879676.921816:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879676.921820:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879676.921827:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:0.0:1788879676.921829:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879676.921833:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1788879676.921835:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879676.921842:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879676.921843:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290187136:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879676.921854:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.921856:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879676.921868:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879676.921871:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879676.921874:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879676.921876:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879676.921879:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879676.921884:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879676.921887:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879676.921892:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879676.921894:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.921896:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.921953:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879676.921956:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.921959:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879676.921961:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879676.921962:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879676.921964:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879676.921966:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879676.921968:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879676.921970:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879676.921973:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879676.921977:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290187264:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879676.921981:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879676.921983:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879676.921985:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879676.921987:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879676.921991:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.921998:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879676.922000:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:2.0:1788879676.922005:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879676.922020:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879676.922025:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879676.922028:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879676.922030:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.922032:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879676.922035:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.922040:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305288204 00000100:00000001:2.0:1788879676.922043:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305288745 00000100:00000001:2.0:1788879676.922046:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305288204 00000100:00000001:2.0:1788879676.922048:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305288745 00000100:00000001:2.0:1788879676.922051:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879676.922053:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.922055:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879676.922059:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879676/real 1788879676] req@ffff94e50897a300 x1875765290187264/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879676.922068:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879676.922070:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879676.922072:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879676.922076:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879676.922078:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922080:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879676.922081:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879676.922085:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290187264/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879676.922092:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922094:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879676.922097:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922098:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879676.922100:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922102:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879676.922119:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879676.922123:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879676.922144:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879676.922148:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879676.922154:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290187264:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879676.922164:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879676.922167:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879676.922170:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879676.922173:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879676.922177:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879676.922180:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879676.922186:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290187392:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879676.922213:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879676.922217:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879676.922220:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879676.922224:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879676.922246:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.922257:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879676.922260:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000100:00000001:2.0:1788879676.922267:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879676.922288:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879676.922294:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879676.922297:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879676.922299:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.922301:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879676.922304:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.922308:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305288202 00000100:00000001:2.0:1788879676.922311:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305288745 00000100:00000001:2.0:1788879676.922314:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305288202 00000100:00000001:2.0:1788879676.922316:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305288745 00000100:00000001:2.0:1788879676.922319:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879676.922321:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.922323:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879676.922328:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879676/real 1788879676] req@ffff94e508978e00 x1875765290187392/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879676.922336:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879676.922338:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879676.922341:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879676.922343:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879676.922344:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922346:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879676.922348:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879676.922352:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290187392/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879676.922359:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922362:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879676.922364:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922366:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879676.922367:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922369:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879676.922385:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879676.922387:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879676.922390:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879676.922392:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879676.922395:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290187392:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879676.922399:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879676.922400:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879676.922402:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879676.922404:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879676.922406:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879676.922409:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879676.922412:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290187520:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879676.922415:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879676.922417:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879676.922419:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879676.922420:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879676.922423:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.922428:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879676.922430:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:2.0:1788879676.922433:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879676.922444:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879676.922448:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879676.922451:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879676.922452:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.922454:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879676.922457:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.922460:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305288202 00000100:00000001:2.0:1788879676.922462:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305288745 00000100:00000001:2.0:1788879676.922465:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305288202 00000100:00000001:2.0:1788879676.922467:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305288745 00000100:00000001:2.0:1788879676.922469:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879676.922471:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.922473:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879676.922477:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879676/real 1788879676] req@ffff94e50897ad80 x1875765290187520/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879676.922484:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879676.922485:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879676.922488:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879676.922489:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879676.922491:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922493:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879676.922494:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879676.922498:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290187520/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879676.922506:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922508:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879676.922510:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922511:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879676.922513:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922514:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879676.922530:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879676.922533:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879676.922536:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879676.922538:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879676.922541:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290187520:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879676.922545:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922547:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879676.922548:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879676.922550:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879676.922552:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879676.922553:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879676.922555:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879676.922558:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879676.922559:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879676.922562:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879676.922563:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922565:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879676.922566:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879676.922568:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879676.922570:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879676.922571:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879676.922572:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879676.922574:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879676.922575:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879676.922577:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879676.922579:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922580:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879676.922581:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879676.922583:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879676.922584:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879676.922585:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879676.922586:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879676.922588:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879676.922589:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879676.922592:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879676.922593:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922595:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879676.922599:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879676.922601:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879676.922603:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879676.922605:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879676.985018:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879676.985027:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879676.985029:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879676.985036:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879676.985041:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879676.985045:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879676.985047:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879676.985050:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879676.985053:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879677.689071:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879677.689081:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879677.689102:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879677.689106:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879677.689127:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879677.689129:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879677.689132:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879677.689135:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879677.689139:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879677.689141:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879677.689287:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879677.689298:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879677.689306:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879678.009029:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879678.009037:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879678.009039:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879678.009045:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879678.009049:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879678.009052:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879678.009055:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879678.009057:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879678.009060:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879678.713105:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879678.713114:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879678.713119:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879678.713120:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879678.713154:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879678.713158:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879678.713162:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879679.033072:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879679.033081:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879679.033083:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879679.033089:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879679.033093:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879679.033097:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879679.033101:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879679.033104:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879679.033107:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879679.737075:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879679.737082:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879679.737087:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879679.737088:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879679.737151:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879679.737156:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879679.737160:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879680.056990:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879680.056998:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879680.057000:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879680.057005:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879680.057009:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879680.057013:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879680.057015:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879680.057018:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879680.057020:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879680.761096:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879680.761108:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879680.761114:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879680.761116:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879680.761255:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879680.761261:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879680.761266:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879681.081022:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879681.081033:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879681.081036:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879681.081044:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879681.081049:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879681.081055:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879681.081060:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879681.081064:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879681.081068:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879681.785034:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879681.785040:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879681.785044:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879681.785046:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879681.785213:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879681.785220:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879681.785227:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879682.041049:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879682.041064:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879682.041068:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879682.041073:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879682.041079:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879682.041088:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10620 00000100:00080000:0.0:1788879682.041103:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879682.041106:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879682.041109:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879682.041119:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879682.041123:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879682.041127:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879682.041130:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879682.041134:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879682.041136:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879682.041142:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879682.041146:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879682.041150:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879682.041174:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879682.041178:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041181:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879682.041185:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879682.041199:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879682.041202:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041206:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879682.041209:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879682.041216:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041247:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879682.041260:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897aa00 x1875765290187648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879682.041274:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879682.041279:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041299:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879682.041304:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879682.041308:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879682.041310:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879682.041313:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879682.041316:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879682.041320:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10620 00000100:00080000:0.0:1788879682.041324:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879682.041326:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879682.041328:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879682.041333:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879682.041334:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879682.041337:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879682.041341:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879682.041344:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879682.041347:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879682.041351:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879682.041354:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879682.041356:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879682.041364:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879682.041368:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041371:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879682.041376:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879682.041381:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041384:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879682.041389:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879682.041391:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041393:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879682.041395:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879682.041398:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041404:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.041410:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1788879682.041412:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978700 x1875765290187776/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879682.041423:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879682.041433:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879682.041440:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879682.041442:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879682.041450:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.041454:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041458:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879682.041471:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1788879682.041476:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1788879682.041477:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879682.041483:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879682.041487:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1788879682.041489:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879682.041491:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879682.041498:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10620 00000100:00080000:0.0:1788879682.041507:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1788879682.041509:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041513:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879682.041520:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879682.041525:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879682.041529:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879682.041532:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00100000:2.0:1788879682.041535:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290187648:192.168.203.129@tcp:250:kworker.0 00000020:00000001:0.0:1788879682.041536:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:0.0:1788879682.041549:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879682.041554:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879682.041557:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1788879682.041564:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:2.0:1788879682.041566:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788879682.041570:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879682.041572:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879682.041575:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879682.041582:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879682.041584:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879682.041590:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041593:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879682.041595:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788879682.041597:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1788879682.041600:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041607:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879682.041613:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879682.041617:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041619:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879682.041621:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879682.041623:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041633:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.041635:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879682.041647:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145419776 : -117763564131840 : ffff94e50897aa00) 00000100:00100000:0.0:1788879682.041657:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978a80 x1875765290187904/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879682.041660:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879682.041675:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879682.041678:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041681:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879682.041685:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879682.041689:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879682.041692:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879682.041694:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879682.041707:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879682.041710:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10620 00000100:00080000:0.0:1788879682.041716:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879682.041718:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879682.041730:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879682.041734:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879682.041735:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879682.041738:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879682.041741:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:0.0:1788879682.041743:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879682.041746:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879682.041750:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879682.041753:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879682.041756:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879682.041761:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879682.041763:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041765:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879682.041768:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879682.041772:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041777:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.041783:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879682.041787:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879682.041793:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305293865 02000000:00000001:0.0:1788879682.041794:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041804:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879682.041808:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879682.041812:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1788879682.041813:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.041828:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305293865 00000100:00000001:0.0:1788879682.041830:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.041839:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879682.041848:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1788879682.041850:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897ad80 x1875765290188032/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879682.041853:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.041863:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.041872:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879682.041889:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1788879682.041891:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879682.041894:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1788879682.041950:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879682.041958:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.041968:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879682.041973:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879682.041977:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879682.041980:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879682.041983:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879682.041988:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879682.041991:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879682.041996:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879682.042004:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290187776:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879682.042009:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879682.042013:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879682.042015:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879682.042017:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879682.042021:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.042032:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879682.042034:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145410816 : -117763564140800 : ffff94e508978700) 00000100:00000001:2.0:1788879682.042055:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879682.042089:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.042096:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305293324 00000100:00000001:2.0:1788879682.042100:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305293865 00000100:00000001:2.0:1788879682.042103:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305293324 00000100:00000001:2.0:1788879682.042107:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305293865 00000100:00000001:2.0:1788879682.042110:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879682.042113:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.042118:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879682.042121:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879682.042123:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879682.042128:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879682.042131:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879682.042134:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879682.042141:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290187904:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879682.042145:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879682.042148:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879682.042151:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879682.042177:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879682.042183:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.042192:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879682.042193:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145411712 : -117763564139904 : ffff94e508978a80) 00000100:00000001:2.0:1788879682.042216:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879682.042230:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.042245:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305293322 00000100:00000001:2.0:1788879682.042248:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305293865 00000100:00000001:2.0:1788879682.042251:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305293322 00000100:00000001:2.0:1788879682.042253:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305293865 00000100:00000001:2.0:1788879682.042256:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879682.042258:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.042260:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879682.042262:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879682.042263:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879682.042266:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879682.042268:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879682.042269:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879682.042274:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290188032:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879682.042277:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879682.042280:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879682.042282:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879682.042284:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879682.042287:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.042291:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879682.042293:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145420672 : -117763564130944 : ffff94e50897ad80) 00000100:00000001:2.0:1788879682.042296:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879682.042305:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.042308:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305293322 00000100:00000001:2.0:1788879682.042312:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305293865 00000100:00000001:2.0:1788879682.042315:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305293322 00000100:00000001:2.0:1788879682.042318:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305293865 00000100:00000001:2.0:1788879682.042322:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879682.042325:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.042329:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.042332:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879682.042337:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879682.042340:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1788879682.042343:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879682.042345:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879682.042348:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879682.042351:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879682.105042:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879682.105049:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879682.105052:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879682.105056:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879682.105069:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879682.105072:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879682.105075:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879682.105077:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879682.105080:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879682.809066:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879682.809074:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879682.809082:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879682.809084:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879682.809140:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879682.809144:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879682.809174:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879683.129007:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879683.129016:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879683.129019:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879683.129026:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879683.129031:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879683.129036:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879683.129040:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879683.129043:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879683.129046:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879683.833105:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879683.833114:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879683.833122:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879683.833124:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879683.833226:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879683.833231:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879683.833236:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879684.153065:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879684.153076:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879684.153080:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879684.153089:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879684.153096:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879684.153102:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879684.153107:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879684.153112:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879684.153117:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879684.857088:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879684.857095:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879684.857100:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879684.857102:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879684.857204:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879684.857211:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879684.857225:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1788879685.113521:0:2351:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:3.0:1788879685.113605:0:2351:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1788879685.113650:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788879685.113689:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879685.113727:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879685.113732:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879685.113737:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:3.0:1788879685.113757:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788879685.113767:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879685.113773:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879685.113777:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879685.113781:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:3.0:1788879685.113795:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788879685.113808:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879685.113813:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879685.113817:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879685.113818:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879685.113829:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879685.113832:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879685.113842:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000800:00000100:3.0:1788879685.113844:0:2351:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:3.0:1788879685.113869:0:2351:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879685.113875:0:2351:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879685.113882:0:2351:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879685.113886:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879682/real 1788879685] req@ffff94e508978700 x1875765290187776/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879685.113887:0:2351:0:(events.c:69:request_out_callback()) Process leaving 00000400:00000001:2.0:1788879685.114003:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879685.114008:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879685.114022:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879685.114029:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879685.114034:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114041:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879685.114046:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879685.114059:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978700 x1875765290187776/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879685.114078:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114086:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879685.114094:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114100:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879685.114103:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114109:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879685.114163:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879685.114173:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879685.114185:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879685.114194:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879685.114203:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290187776:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879685.114230:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879685.114241:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879682/real 1788879685] req@ffff94e508978a80 x1875765290187904/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879685.114259:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879685.114262:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879685.114269:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879685.114272:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879685.114275:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114280:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879685.114282:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879685.114290:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978a80 x1875765290187904/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879685.114305:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114309:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879685.114314:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114317:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879685.114319:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114322:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879685.114345:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879685.114378:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879685.114387:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879685.114392:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879685.114401:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290187904:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879685.114409:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879685.114418:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879682/real 1788879685] req@ffff94e50897ad80 x1875765290188032/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879685.114436:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879685.114439:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879685.114445:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879685.114449:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879685.114451:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114455:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879685.114458:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879685.114466:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897ad80 x1875765290188032/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879685.114478:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114483:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879685.114487:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114491:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879685.114493:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114497:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879685.114520:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879685.114526:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879685.114533:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879685.114550:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879685.114557:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290188032:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879685.114566:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879685.114574:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879682/real 1788879685] req@ffff94e50897aa00 x1875765290187648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879685.114589:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879685.114592:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879685.114599:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879685.114602:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879685.114604:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114608:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879685.114610:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879685.114617:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897aa00 x1875765290187648/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879685.114629:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114633:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879685.114637:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114640:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879685.114642:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114645:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879685.114648:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879685.114652:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879685.114659:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879685.114662:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879685.114668:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290187648:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879685.114676:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114682:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879685.114685:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879685.114690:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879685.114700:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879685.114705:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879685.114710:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879685.114718:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879685.114722:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879685.114729:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879685.114732:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114735:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879685.114738:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879685.114741:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879685.114745:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879685.114747:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879685.114750:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879685.114755:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879685.114757:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879685.114760:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879685.114762:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114765:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879685.114768:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879685.114770:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879685.114773:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879685.114775:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879685.114777:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879685.114781:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879685.114783:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879685.114790:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879685.114793:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114797:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879685.114799:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879685.114801:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879685.114805:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879685.114806:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879685.114809:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879685.114812:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879685.114815:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879685.114818:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879685.114820:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114824:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879685.114838:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879685.114841:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879685.114845:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879685.114849:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879685.177015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879685.177029:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879685.177033:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879685.177042:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879685.177049:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879685.177055:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879685.177060:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879685.177065:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879685.177071:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879685.881060:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879685.881068:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879685.881075:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879685.881079:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879685.881093:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879685.881098:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879685.881101:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879685.881104:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879685.881107:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879685.881110:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879685.881238:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879685.881255:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879685.881261:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879686.201021:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879686.201029:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879686.201031:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879686.201037:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879686.201041:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879686.201045:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879686.201048:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879686.201051:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879686.201054:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879686.905019:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879686.905028:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879686.905034:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879686.905036:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879686.905090:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879686.905093:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879686.905096:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879687.161060:0:178259:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879687.161075:0:178259:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879687.161079:0:178259:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879687.161084:0:178259:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879687.161091:0:178259:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879687.161102:0:178259:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10626 00000100:00080000:2.0:1788879687.161138:0:178259:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879687.161155:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879687.161157:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879687.161170:0:178259:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879687.161174:0:178259:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879687.161179:0:178259:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879687.161182:0:178259:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879687.161185:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879687.161187:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879687.161193:0:178259:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879687.161196:0:178259:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879687.161201:0:178259:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879687.161205:0:178259:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879687.161209:0:178259:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879687.161212:0:178259:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879687.161215:0:178259:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879687.161229:0:178259:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879687.161231:0:178259:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879687.161236:0:178259:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879687.161239:0:178259:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879687.161245:0:178259:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879687.161262:0:178259:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879687.161275:0:178259:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6300 x1875765290188160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879687.161289:0:178259:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879687.161295:0:178259:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879687.161321:0:178259:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879687.161328:0:178259:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879687.161336:0:178259:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879687.161338:0:178259:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879687.161342:0:178259:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879687.161347:0:178259:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879687.161352:0:178259:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10626 00000100:00080000:2.0:1788879687.161361:0:178259:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879687.161365:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879687.161368:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879687.161375:0:178259:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879687.161377:0:178259:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879687.161381:0:178259:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879687.161386:0:178259:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879687.161390:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879687.161393:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879687.161400:0:178259:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879687.161402:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879687.161405:0:178259:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879687.161408:0:178259:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879687.161410:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879687.161414:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788879687.161417:0:178259:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879687.161419:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879687.161422:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879687.161422:0:178259:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879687.161426:0:178259:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879687.161430:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:2.0:1788879687.161433:0:178259:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879687.161435:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1788879687.161438:0:178259:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879687.161439:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879687.161442:0:178259:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879687.161448:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290188160:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879687.161450:0:178259:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879687.161453:0:178259:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879687.161456:0:178259:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879687.161457:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879687.161460:0:178259:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879687.161463:0:178259:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879687.161464:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879687.161469:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879687.161474:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879687.161474:0:178259:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879687.161484:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879687.161484:0:178259:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5f80 x1875765290188288/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879687.161498:0:178259:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879687.161502:0:178259:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879687.161507:0:178259:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.161512:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788879687.161512:0:178259:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879687.161515:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074557184 : -117763634994432 : ffff94e5045e6300) 00000100:00080000:2.0:1788879687.161519:0:178259:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879687.161522:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879687.161522:0:178259:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879687.161526:0:178259:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879687.161531:0:178259:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879687.161535:0:178259:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10626 00000100:00080000:2.0:1788879687.161541:0:178259:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879687.161545:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879687.161547:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879687.161552:0:178259:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879687.161554:0:178259:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879687.161558:0:178259:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879687.161562:0:178259:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879687.161566:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879687.161568:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879687.161574:0:178259:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:1.0:1788879687.161583:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879687.161590:0:178259:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879687.161593:0:178259:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879687.161598:0:178259:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879687.161601:0:178259:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.161604:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879687.161605:0:178259:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879687.161609:0:178259:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879687.161611:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00000001:2.0:1788879687.161614:0:178259:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.161615:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879687.161617:0:178259:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.161619:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:2.0:1788879687.161626:0:178259:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879687.161628:0:178259:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.161629:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879687.161632:0:178259:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879687.161634:0:178259:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879687.161637:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305298985 02000000:00000001:2.0:1788879687.161638:0:178259:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879687.161647:0:178259:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.161653:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00100000:2.0:1788879687.161657:0:178259:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7800 x1875765290188416/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879687.161671:0:178259:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879687.161683:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305298985 00000100:00000001:1.0:1788879687.161702:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879687.161702:0:178259:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.161709:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879687.161709:0:178259:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.161715:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879687.161732:0:178259:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879687.161737:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1788879687.161739:0:178259:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879687.161742:0:178259:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879687.161746:0:178259:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788879687.161747:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879687/real 1788879687] req@ffff94e5045e6300 x1875765290188160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879687.161752:0:178259:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879687.161759:0:178259:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10626 00000100:00080000:2.0:1788879687.161765:0:178259:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000400:00000001:1.0:1788879687.161767:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1788879687.161769:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000400:00000001:1.0:1788879687.161770:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879687.161772:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879687.161777:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000020:00000001:2.0:1788879687.161778:0:178259:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879687.161780:0:178259:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879687.161782:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000020:00000001:2.0:1788879687.161784:0:178259:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879687.161785:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788879687.161788:0:178259:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879687.161790:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879687.161792:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879687.161793:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879687.161794:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879687.161800:0:178259:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1788879687.161802:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6300 x1875765290188160/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879687.161805:0:178259:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879687.161809:0:178259:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879687.161814:0:178259:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879687.161816:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879687.161818:0:178259:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879687.161821:0:178259:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879687.161822:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00100000:2.0:1788879687.161825:0:178259:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879687.161828:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1788879687.161829:0:178259:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.161831:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879687.161832:0:178259:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.161833:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.161838:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1788879687.161839:0:178259:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879687.161842:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788879687.161842:0:178259:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879687.161846:0:178259:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:1.0:1788879687.161848:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1788879687.161848:0:178259:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879687.161852:0:178259:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879687.161855:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879687.161860:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788879687.161861:0:178259:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879687.161868:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290188160:192.168.203.129@tcp:250:kworker.0 00000100:00100000:2.0:1788879687.161869:0:178259:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4380 x1875765290188544/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879687.161877:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.161881:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879687.161884:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788879687.161884:0:178259:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879687.161888:0:178259:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879687.161889:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788879687.161892:0:178259:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879687.161895:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879687.161901:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879687.161904:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879687.161910:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879687.161913:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879687.161961:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879687.161965:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.161970:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.161982:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879687.161986:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.161990:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879687.161993:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879687.161996:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879687.161998:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879687.162001:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879687.162005:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879687.162009:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879687.162013:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879687.162020:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290188288:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879687.162026:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879687.162030:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879687.162033:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879687.162036:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879687.162042:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.162054:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879687.162057:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074556288 : -117763634995328 : ffff94e5045e5f80) 00000100:00000001:1.0:1788879687.162065:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879687.162092:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879687.162102:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879687.162107:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879687.162110:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.162113:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879687.162141:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.162150:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305298444 00000100:00000001:1.0:1788879687.162155:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305298985 00000100:00000001:1.0:1788879687.162160:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305298444 00000100:00000001:1.0:1788879687.162164:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305298985 00000100:00000001:1.0:1788879687.162169:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879687.162173:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.162176:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879687.162186:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879687/real 1788879687] req@ffff94e5045e5f80 x1875765290188288/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879687.162203:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879687.162206:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879687.162212:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879687.162215:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879687.162218:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162221:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879687.162224:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879687.162231:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5f80 x1875765290188288/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879687.162254:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162258:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879687.162263:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162269:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879687.162272:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162275:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879687.162298:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879687.162303:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879687.162311:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879687.162314:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879687.162321:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290188288:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879687.162329:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879687.162332:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879687.162334:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879687.162338:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879687.162341:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879687.162345:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879687.162352:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290188416:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879687.162358:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879687.162361:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879687.162364:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879687.162367:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879687.162372:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.162382:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879687.162385:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074562560 : -117763634989056 : ffff94e5045e7800) 00000100:00000001:1.0:1788879687.162391:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879687.162416:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879687.162424:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879687.162428:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879687.162430:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.162434:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879687.162438:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.162444:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305298442 00000100:00000001:1.0:1788879687.162448:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305298985 00000100:00000001:1.0:1788879687.162453:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305298442 00000100:00000001:1.0:1788879687.162457:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305298985 00000100:00000001:1.0:1788879687.162460:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879687.162465:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.162468:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879687.162476:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879687/real 1788879687] req@ffff94e5045e7800 x1875765290188416/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879687.162491:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879687.162494:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879687.162500:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879687.162503:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879687.162505:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162508:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879687.162511:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879687.162535:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7800 x1875765290188416/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879687.162558:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162562:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879687.162566:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162569:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879687.162571:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162574:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879687.162593:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879687.162598:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879687.162603:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879687.162607:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879687.162612:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290188416:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879687.162619:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879687.162621:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879687.162623:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879687.162627:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879687.162630:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879687.162633:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879687.162638:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290188544:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879687.162644:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879687.162647:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879687.162649:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879687.162652:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879687.162658:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.162667:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879687.162670:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074549120 : -117763635002496 : ffff94e5045e4380) 00000100:00000001:1.0:1788879687.162675:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879687.162698:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879687.162706:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879687.162709:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879687.162712:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.162715:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879687.162720:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.162725:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305298442 00000100:00000001:1.0:1788879687.162729:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305298985 00000100:00000001:1.0:1788879687.162733:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305298442 00000100:00000001:1.0:1788879687.162736:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305298985 00000100:00000001:1.0:1788879687.162739:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879687.162743:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.162746:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879687.162753:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879687/real 1788879687] req@ffff94e5045e4380 x1875765290188544/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879687.162765:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879687.162767:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879687.162772:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879687.162774:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879687.162777:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162780:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879687.162782:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879687.162789:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4380 x1875765290188544/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879687.162800:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162804:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879687.162808:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162810:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879687.162812:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162815:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879687.162833:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879687.162838:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879687.162844:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879687.162848:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879687.162854:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290188544:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879687.162869:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162873:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879687.162875:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879687.162878:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879687.162883:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879687.162885:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879687.162888:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879687.162892:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879687.162894:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879687.162897:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879687.162899:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162937:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879687.162940:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879687.162942:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879687.162947:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879687.162949:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879687.162950:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879687.162954:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879687.162956:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879687.162959:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879687.162961:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162964:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879687.162966:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879687.162968:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879687.162970:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879687.162972:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879687.162974:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879687.162977:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879687.162978:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879687.162981:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879687.162983:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162987:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.162994:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879687.162997:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.163001:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879687.163004:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879687.225033:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879687.225041:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879687.225043:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879687.225051:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879687.225056:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879687.225060:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.225064:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879687.225068:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879687.225071:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879687.929072:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879687.929084:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879687.929088:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879687.929092:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879687.929100:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879687.929104:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879687.929106:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879687.929109:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879687.929134:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879687.929136:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879687.929167:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879687.929171:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879687.929175:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879688.249024:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879688.249035:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879688.249038:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879688.249046:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879688.249051:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879688.249056:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879688.249060:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879688.249064:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879688.249067:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879688.953010:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879688.953017:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879688.953021:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879688.953022:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879688.953074:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879688.953078:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879688.953082:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879689.273024:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879689.273029:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879689.273031:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879689.273035:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879689.273038:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879689.273041:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879689.273043:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879689.273046:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879689.273048:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879689.977150:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879689.977161:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879689.977168:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879689.977170:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879689.977251:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879689.977258:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879689.977263:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879690.297053:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879690.297062:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879690.297065:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879690.297072:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879690.297076:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879690.297082:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879690.297086:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879690.297090:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879690.297093:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879691.001067:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879691.001077:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879691.001084:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879691.001086:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879691.001200:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879691.001205:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879691.001210:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879691.321032:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879691.321041:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879691.321044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879691.321051:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879691.321056:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879691.321061:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879691.321065:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879691.321069:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879691.321072:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879692.025100:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879692.025115:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879692.025121:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879692.025122:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879692.025150:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879692.025152:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879692.025155:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879692.281053:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879692.281065:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879692.281128:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879692.281151:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879692.281156:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879692.281164:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10631 00000100:00080000:3.0:1788879692.281175:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879692.281177:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879692.281179:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879692.281186:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879692.281188:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879692.281191:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879692.281192:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879692.281194:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879692.281195:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879692.281199:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879692.281201:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879692.281203:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879692.281207:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879692.281209:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281211:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879692.281213:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879692.281222:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879692.281224:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281227:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879692.281229:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879692.281234:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281250:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879692.281257:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290188672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879692.281265:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879692.281269:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281282:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879692.281284:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879692.281287:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879692.281288:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879692.281290:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879692.281292:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879692.281294:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10631 00000100:00080000:3.0:1788879692.281296:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879692.281298:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879692.281299:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879692.281301:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879692.281302:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879692.281304:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879692.281306:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879692.281308:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879692.281309:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879692.281312:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879692.281313:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879692.281314:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879692.281318:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879692.281319:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788879692.281322:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879692.281323:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879692.281327:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788879692.281327:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.281330:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281331:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879692.281336:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879692.281337:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879692.281341:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1788879692.281341:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281343:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879692.281345:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879692.281348:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879692.281351:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879692.281352:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290188672:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879692.281353:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879692.281355:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879692.281359:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879692.281359:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281363:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879692.281364:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879692.281368:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879692.281371:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1788879692.281372:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290188800/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879692.281375:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281382:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879692.281383:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281386:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879692.281387:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879692.281389:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879692.281390:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879692.281391:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879692.281393:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879692.281394:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788879692.281396:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10631 00000100:00000001:1.0:1788879692.281397:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00080000:3.0:1788879692.281399:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879692.281402:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879692.281403:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879692.281406:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879692.281409:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879692.281409:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879692.281411:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879692.281412:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879692.281414:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879692.281415:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879692.281418:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879692.281419:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879692.281421:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879692.281423:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879692.281425:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281426:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879692.281428:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879692.281430:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281431:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879692.281435:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879692.281436:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879692.281439:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879692.281440:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879692.281443:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879692.281444:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281448:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.281453:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00100000:3.0:1788879692.281453:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290188928/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879692.281457:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879692.281461:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281462:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879692.281466:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879692.281467:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281471:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.281472:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879692.281473:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879692.281477:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305304105 00000100:00080000:3.0:1788879692.281477:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879692.281481:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879692.281483:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879692.281485:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879692.281487:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10631 00000100:00000001:1.0:1788879692.281489:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788879692.281490:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879692.281494:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879692.281495:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879692.281497:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879692.281498:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879692.281499:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305304105 00000020:00000001:3.0:1788879692.281500:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879692.281504:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879692.281516:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879692.281517:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879692.281518:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879692.281522:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879692.281523:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879692.281527:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281528:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879692.281532:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879692.281533:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1788879692.281535:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879692.281538:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281540:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00100000:1.0:1788879692.281542:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879692/real 1788879692] req@ffff94e50449d880 x1875765290188672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1788879692.281543:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879692.281548:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281551:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879692.281557:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879692.281559:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281564:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879692.281566:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879692.281569:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281575:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879692.281577:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879692.281579:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:3.0:1788879692.281583:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290189056/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879692.281584:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879692.281588:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879692.281590:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.281595:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879692.281595:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879692.281598:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879692.281599:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879692.281603:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879692.281607:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290188672/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879692.281613:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.281616:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879692.281618:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.281619:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879692.281620:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.281622:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879692.281624:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879692.281626:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879692.281629:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879692.281630:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879692.281633:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290188672:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879692.281636:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.281638:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879692.281639:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879692.281640:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879692.281643:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879692.281644:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879692.281646:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879692.281648:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879692.281650:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879692.281651:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879692.281652:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.281654:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.281660:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879692.281662:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.281664:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879692.281665:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879692.281666:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879692.281667:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879692.281668:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879692.281669:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879692.281671:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879692.281672:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879692.281674:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290188800:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879692.281677:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879692.281678:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879692.281679:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879692.281680:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879692.281683:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.281687:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879692.281688:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:1.0:1788879692.281691:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879692.281699:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879692.281702:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879692.281703:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879692.281704:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.281706:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879692.281707:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.281711:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305303563 00000100:00000001:1.0:1788879692.281713:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305304105 00000100:00000001:1.0:1788879692.281715:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305303563 00000100:00000001:1.0:1788879692.281716:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305304105 00000100:00000001:1.0:1788879692.281718:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879692.281719:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.281720:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879692.281723:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879692/real 1788879692] req@ffff94e50449fb80 x1875765290188800/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879692.281733:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879692.281734:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879692.281736:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879692.281737:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879692.281737:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.281738:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879692.281739:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879692.281741:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290188800/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879692.281746:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.281747:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879692.281749:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.281750:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879692.281751:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.281753:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879692.281782:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879692.281784:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879692.281786:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879692.281788:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879692.281791:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290188800:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879692.281795:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879692.281796:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879692.281797:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879692.281798:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879692.281800:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879692.281801:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879692.281803:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290188928:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879692.281805:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879692.281807:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879692.281809:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879692.281810:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879692.281812:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.281817:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879692.281818:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:1.0:1788879692.281821:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879692.281829:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879692.281832:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879692.281834:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879692.281835:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.281836:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879692.281838:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.281841:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305303561 00000100:00000001:1.0:1788879692.281844:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305304105 00000100:00000001:1.0:1788879692.281846:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305303561 00000100:00000001:1.0:1788879692.281848:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305304105 00000100:00000001:1.0:1788879692.281850:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879692.281851:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.281853:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879692.281855:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879692/real 1788879692] req@ffff94e50449d180 x1875765290188928/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879692.281861:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879692.281862:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879692.281863:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879692.281864:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879692.281865:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.281867:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879692.281868:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879692.281870:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290188928/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879692.281876:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.281877:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879692.281880:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.281881:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879692.281882:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.281884:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879692.281897:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879692.281900:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879692.281902:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879692.281904:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879692.281906:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290188928:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879692.281909:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879692.281910:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879692.281911:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879692.281912:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879692.281914:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879692.281965:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879692.281973:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290189056:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879692.281979:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879692.281982:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879692.281985:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879692.281988:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879692.281994:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.282004:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879692.282022:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:1.0:1788879692.282026:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879692.282041:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879692.282046:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879692.282049:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879692.282051:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.282053:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879692.282057:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.282061:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305303562 00000100:00000001:1.0:1788879692.282064:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305304105 00000100:00000001:1.0:1788879692.282067:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305303562 00000100:00000001:1.0:1788879692.282069:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305304105 00000100:00000001:1.0:1788879692.282071:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879692.282073:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.282075:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879692.282078:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879692/real 1788879692] req@ffff94e50449ce00 x1875765290189056/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879692.282104:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879692.282105:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879692.282108:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879692.282109:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879692.282111:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.282113:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879692.282114:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879692.282117:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290189056/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879692.282149:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.282151:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879692.282152:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.282153:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879692.282154:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.282155:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879692.282166:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879692.282168:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879692.282170:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879692.282171:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879692.282173:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290189056:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879692.282175:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.282177:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879692.282179:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879692.282180:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879692.282182:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879692.282183:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879692.282184:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879692.282185:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879692.282186:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879692.282188:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879692.282189:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.282190:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879692.282191:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879692.282192:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879692.282193:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879692.282194:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879692.282195:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879692.282196:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879692.282197:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879692.282197:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879692.282198:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.282199:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879692.282200:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879692.282200:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879692.282202:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879692.282203:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879692.282203:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879692.282204:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879692.282205:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879692.282206:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879692.282206:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.282207:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.282210:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879692.282210:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.282212:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879692.282213:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879692.344967:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879692.344985:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879692.344987:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879692.344992:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879692.344995:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879692.344998:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.345001:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879692.345004:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879692.345007:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879693.049025:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879693.049033:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879693.049041:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879693.049043:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879693.049202:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879693.049208:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879693.049213:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879693.369007:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879693.369015:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879693.369018:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879693.369023:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879693.369027:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879693.369032:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879693.369036:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879693.369038:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879693.369041:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879694.073094:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879694.073102:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879694.073110:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879694.073122:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879694.073240:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879694.073251:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879694.073262:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879694.393015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879694.393024:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879694.393028:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879694.393034:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879694.393039:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879694.393044:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879694.393047:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879694.393050:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879694.393053:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879695.097100:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879695.097110:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879695.097118:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879695.097122:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879695.097125:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879695.097129:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879695.097133:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879695.097135:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879695.097138:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879695.097140:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879695.097169:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879695.097172:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879695.097175:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879695.417018:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879695.417041:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879695.417043:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879695.417051:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879695.417057:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879695.417079:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879695.417083:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879695.417087:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879695.417090:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879696.121066:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879696.121076:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879696.121082:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879696.121084:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879696.121102:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879696.121105:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879696.121107:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879696.441020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879696.441030:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879696.441032:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879696.441038:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879696.441068:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879696.441072:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879696.441075:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879696.441077:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879696.441080:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879697.145042:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879697.145068:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879697.145072:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879697.145077:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879697.145083:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879697.145086:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879697.145089:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879697.145092:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879697.145110:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879697.145113:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879697.145196:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879697.145201:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879697.145205:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879697.401103:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879697.401123:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879697.401128:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879697.401135:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879697.401145:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879697.401158:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10636 00000100:00080000:3.0:1788879697.401179:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879697.401185:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879697.401190:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879697.401205:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879697.401210:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879697.401217:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879697.401234:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879697.401239:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879697.401243:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879697.401254:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879697.401261:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879697.401268:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879697.401276:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879697.401283:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879697.401289:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879697.401294:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879697.401310:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879697.401315:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879697.401321:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879697.401325:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879697.401334:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879697.401354:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879697.401368:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290189184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879697.401383:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879697.401389:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879697.401414:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879697.401419:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879697.401424:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879697.401427:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879697.401429:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879697.401434:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879697.401438:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10636 00000100:00080000:3.0:1788879697.401446:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879697.401449:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879697.401451:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879697.401457:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879697.401461:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879697.401465:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879697.401479:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879697.401482:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879697.401485:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879697.401491:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879697.401493:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879697.401495:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879697.401503:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879697.401508:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788879697.401512:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879697.401516:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788879697.401519:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879697.401525:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879697.401527:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1788879697.401531:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879697.401535:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788879697.401538:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879697.401549:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879697.401556:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879697.401563:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879697.401576:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290189184:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879697.401588:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879697.401595:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879697.401599:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879697.401603:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879697.401610:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.401643:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879697.401646:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:1.0:1788879697.401653:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879697.401685:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879697.401695:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879697.401698:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879697.401701:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879697.401703:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879697.401707:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879697.401718:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879697.401721:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879697.401734:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290189312/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879697.401741:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879697.401751:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879697.401755:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879697.401757:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.401768:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879697.401769:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879697.401779:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879697.401782:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:3.0:1788879697.401786:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879697.401791:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879697.401794:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879697.401797:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879697.401800:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879697.401808:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10636 00000100:00000001:1.0:1788879697.401809:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305309225 00000100:00080000:3.0:1788879697.401814:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879697.401820:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879697.401823:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879697.401827:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1788879697.401829:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879697.401836:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879697.401840:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879697.401843:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879697.401846:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879697.401848:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879697.401854:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305309225 00000100:00080000:3.0:1788879697.401855:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879697.401871:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879697.401879:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879697.401885:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.401890:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879697.401953:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879697/real 1788879697] req@ffff94e50449df80 x1875765290189184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879697.401991:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879697.401995:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879697.402004:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879697.402010:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879697.402014:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.402020:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879697.402024:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879697.402036:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290189184/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879697.402088:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.402094:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879697.402098:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.402100:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879697.402102:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.402106:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879697.402109:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879697.402113:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879697.402119:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879697.402123:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879697.402130:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290189184:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879697.402133:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879697.402143:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879697.402145:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879697.402148:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879697.402152:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788879697.402158:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.402160:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1788879697.402164:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879697.402167:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000008:00100000:3.0:1788879697.402170:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879697.402175:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000008:00000001:3.0:1788879697.402177:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879697.402182:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:3.0:1788879697.402184:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879697.402189:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:3.0:1788879697.402196:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879697.402197:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1788879697.402202:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879697.402204:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879697.402211:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879697.402215:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 02000000:00000001:3.0:1788879697.402216:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879697.402222:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879697.402224:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.402241:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879697.402244:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.402256:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1788879697.402259:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290189440/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879697.402266:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879697.402280:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879697.402285:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879697.402291:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.402307:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879697.402315:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879697.402318:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879697.402321:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879697.402325:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879697.402331:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879697.402335:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879697.402340:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879697.402349:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290189312:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879697.402359:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788879697.402364:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879697.402365:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879697.402373:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879697.402375:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000001:1.0:1788879697.402384:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879697.402384:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879697.402392:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1788879697.402396:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879697.402399:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879697.402407:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10636 00000100:00000001:1.0:1788879697.402411:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788879697.402414:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879697.402417:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000100:00000001:3.0:1788879697.402420:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879697.402425:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879697.402430:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1788879697.402433:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879697.402438:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879697.402441:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879697.402444:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879697.402448:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879697.402451:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879697.402457:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879697.402463:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879697.402466:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879697.402467:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879697.402485:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879697.402491:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788879697.402492:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879697.402500:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879697.402502:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00100000:3.0:1788879697.402504:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879697.402510:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788879697.402512:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879697.402521:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.402523:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879697.402531:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879697.402532:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879697.402538:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.402543:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305308684 00000100:00000001:3.0:1788879697.402545:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879697.402550:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879697.402555:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305309225 02000000:00000001:3.0:1788879697.402556:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.402566:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305308684 00000100:00000001:3.0:1788879697.402568:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.402572:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305309225 00000100:00000001:1.0:1788879697.402580:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788879697.402582:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290189568/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879697.402587:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.402598:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879697.402606:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1788879697.402613:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879697/real 1788879697] req@ffff94e50449e680 x1875765290189312/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879697.402614:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879697.402635:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879697.402650:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879697.402653:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879697.402657:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879697.402661:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879697.402664:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.402667:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879697.402670:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879697.402676:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290189312/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879697.402687:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.402689:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879697.402692:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.402694:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879697.402696:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.402699:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879697.402719:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879697.402722:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879697.402726:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879697.402729:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879697.402734:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290189312:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879697.402738:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879697.402740:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879697.402742:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879697.402744:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879697.402747:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879697.402750:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879697.402753:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290189440:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879697.402758:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879697.402760:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879697.402763:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879697.402765:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879697.402769:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.402775:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879697.402776:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:1.0:1788879697.402780:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879697.402793:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879697.402800:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879697.402803:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879697.402805:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.402807:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879697.402810:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.402814:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305308682 00000100:00000001:1.0:1788879697.402818:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305309225 00000100:00000001:1.0:1788879697.402821:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305308682 00000100:00000001:1.0:1788879697.402824:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305309225 00000100:00000001:1.0:1788879697.402826:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879697.402829:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.402831:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879697.402835:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879697/real 1788879697] req@ffff94e50449ce00 x1875765290189440/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879697.402844:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879697.402846:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879697.402848:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879697.402850:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879697.402851:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.402853:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879697.402855:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879697.402859:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290189440/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879697.402868:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.402870:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879697.402873:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.402875:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879697.402877:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.402879:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879697.402946:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879697.402952:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879697.402958:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879697.402961:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879697.402967:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290189440:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879697.402973:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.402975:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879697.402977:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879697.402979:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879697.402983:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879697.402985:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879697.402987:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879697.402990:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879697.402992:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879697.402995:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879697.402997:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.403000:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879697.403002:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879697.403004:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879697.403007:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879697.403009:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879697.403012:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879697.403014:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879697.403016:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879697.403018:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879697.403020:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.403022:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.403027:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879697.403028:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.403031:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879697.403033:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879697.403035:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879697.403038:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879697.403041:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879697.403072:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879697.403077:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879697.403081:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879697.403087:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290189568:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879697.403095:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879697.403099:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879697.403102:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879697.403106:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879697.403112:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.403124:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879697.403127:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:1.0:1788879697.403134:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879697.403157:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879697.403167:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879697.403172:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879697.403175:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.403178:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879697.403184:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.403190:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305308683 00000100:00000001:1.0:1788879697.403196:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305309225 00000100:00000001:1.0:1788879697.403200:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305308683 00000100:00000001:1.0:1788879697.403204:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305309225 00000100:00000001:1.0:1788879697.403209:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879697.403213:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.403218:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879697.403226:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879697/real 1788879697] req@ffff94e50449d180 x1875765290189568/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879697.403243:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879697.403246:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879697.403252:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879697.403255:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879697.403260:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.403263:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879697.403266:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879697.403273:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290189568/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879697.403286:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.403290:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879697.403296:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.403299:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879697.403301:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.403305:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879697.403329:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879697.403335:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879697.403341:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879697.403346:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879697.403352:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290189568:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879697.403360:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.403365:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879697.403367:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879697.403370:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879697.403375:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879697.403378:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879697.403380:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879697.403383:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879697.403384:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879697.403387:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879697.403389:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.403391:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.403395:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879697.403397:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.403400:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879697.403402:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879697.465020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879697.465029:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879697.465032:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879697.465040:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879697.465071:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879697.465078:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.465094:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879697.465098:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879697.465104:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879698.169130:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879698.169141:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879698.169148:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879698.169149:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879698.169229:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879698.169234:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879698.169238:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879698.489033:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879698.489065:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879698.489070:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879698.489087:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879698.489092:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879698.489097:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879698.489101:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879698.489105:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879698.489108:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879699.193020:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879699.193031:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879699.193048:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879699.193050:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879699.193114:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879699.193118:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879699.193122:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879699.513019:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879699.513050:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879699.513055:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879699.513064:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879699.513071:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879699.513079:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879699.513084:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879699.513089:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879699.513094:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879700.217021:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879700.217045:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879700.217053:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879700.217055:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879700.217087:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879700.217090:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879700.217094:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879700.537086:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879700.537095:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879700.537098:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879700.537104:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879700.537108:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879700.537113:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879700.537116:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879700.537120:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879700.537123:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879701.241144:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879701.241175:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879701.241185:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879701.241188:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879701.241292:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879701.241298:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879701.241304:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879701.561000:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879701.561008:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879701.561011:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879701.561034:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879701.561039:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879701.561044:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879701.561048:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879701.561051:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879701.561071:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879702.265044:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879702.265053:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879702.265059:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879702.265061:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879702.265111:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879702.265121:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879702.265125:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879702.521044:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879702.521055:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879702.521059:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879702.521062:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879702.521066:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879702.521072:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10641 00000100:00080000:3.0:1788879702.521081:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879702.521089:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879702.521091:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879702.521109:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879702.521111:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879702.521114:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879702.521116:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879702.521117:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879702.521118:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879702.521122:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879702.521124:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879702.521126:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879702.521130:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879702.521133:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879702.521134:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879702.521136:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879702.521146:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879702.521147:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879702.521150:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879702.521152:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879702.521157:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879702.521177:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879702.521185:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290189696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879702.521193:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879702.521196:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879702.521208:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879702.521211:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879702.521213:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879702.521215:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879702.521216:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879702.521218:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879702.521220:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10641 00000100:00080000:3.0:1788879702.521222:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879702.521223:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879702.521224:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879702.521227:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879702.521228:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879702.521229:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879702.521231:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879702.521233:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879702.521234:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879702.521236:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879702.521238:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879702.521238:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879702.521243:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879702.521244:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879702.521248:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788879702.521248:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879702.521252:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788879702.521253:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521256:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879702.521257:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879702.521262:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879702.521263:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879702.521267:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1788879702.521267:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879702.521272:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879702.521273:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879702.521279:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879702.521280:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879702.521281:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290189696:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879702.521282:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879702.521285:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879702.521288:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879702.521288:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879702.521293:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788879702.521295:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879702.521296:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879702.521300:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1788879702.521303:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290189824/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879702.521304:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879702.521317:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879702.521318:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521321:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879702.521321:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521324:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00080000:3.0:1788879702.521326:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879702.521331:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879702.521331:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879702.521332:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879702.521335:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879702.521337:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879702.521339:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10641 00000100:00080000:3.0:1788879702.521341:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879702.521342:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879702.521343:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879702.521345:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879702.521346:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879702.521348:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879702.521349:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879702.521350:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879702.521351:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879702.521353:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879702.521355:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879702.521357:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879702.521358:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879702.521360:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879702.521361:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879702.521363:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879702.521366:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879702.521367:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879702.521369:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879702.521374:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879702.521375:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879702.521377:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879702.521378:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879702.521380:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788879702.521380:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521386:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879702.521386:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521390:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521394:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:3.0:1788879702.521394:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290189952/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879702.521400:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879702.521401:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879702.521404:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521405:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305314345 00000100:00000001:3.0:1788879702.521406:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879702.521411:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879702.521413:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879702.521414:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879702.521415:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879702.521416:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879702.521417:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788879702.521419:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10641 00000100:00080000:3.0:1788879702.521424:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879702.521425:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879702.521426:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305314345 00000100:00000001:3.0:1788879702.521427:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879702.521433:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:3.0:1788879702.521433:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879702.521434:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879702.521437:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879702.521438:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788879702.521439:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879702.521442:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879702.521443:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879702.521447:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879702.521447:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879702.521451:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00100000:1.0:1788879702.521455:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879702/real 1788879702] req@ffff94e50449fb80 x1875765290189696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879702.521455:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879702.521460:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879702.521463:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879702.521467:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879702.521470:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879702.521473:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000400:00000001:1.0:1788879702.521476:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:3.0:1788879702.521477:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879702.521478:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879702.521480:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521482:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879702.521485:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:3.0:1788879702.521485:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879702.521488:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521489:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879702.521491:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879702.521492:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788879702.521493:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879702.521496:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879702.521497:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879702.521501:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290189696/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879702.521505:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521509:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521513:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879702.521516:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521517:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:3.0:1788879702.521517:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290190080/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879702.521519:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521524:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879702.521526:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879702.521531:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879702.521538:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879702.521538:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879702.521542:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788879702.521544:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879702.521547:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290189696:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879702.521549:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521552:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521555:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879702.521556:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879702.521557:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879702.521560:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879702.521562:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879702.521563:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879702.521566:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879702.521567:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879702.521569:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879702.521570:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521572:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521578:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879702.521580:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521582:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879702.521583:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879702.521584:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879702.521585:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879702.521586:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879702.521587:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879702.521588:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879702.521590:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879702.521592:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290189824:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879702.521595:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879702.521596:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879702.521597:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879702.521599:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879702.521600:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521604:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879702.521605:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:1.0:1788879702.521609:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879702.521616:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879702.521619:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879702.521621:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879702.521622:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521623:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879702.521625:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521628:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305313803 00000100:00000001:1.0:1788879702.521630:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305314345 00000100:00000001:1.0:1788879702.521632:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305313803 00000100:00000001:1.0:1788879702.521633:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305314345 00000100:00000001:1.0:1788879702.521635:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879702.521636:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521637:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879702.521639:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879702/real 1788879702] req@ffff94e50449d880 x1875765290189824/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879702.521645:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879702.521646:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879702.521647:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879702.521648:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879702.521649:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521650:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879702.521651:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879702.521654:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290189824/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879702.521659:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521660:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879702.521662:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521663:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879702.521664:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521665:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879702.521678:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879702.521679:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879702.521681:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879702.521682:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879702.521684:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290189824:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879702.521687:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879702.521688:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879702.521689:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879702.521690:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879702.521691:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879702.521693:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879702.521694:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290189952:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879702.521696:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879702.521697:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879702.521698:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879702.521699:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879702.521701:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521703:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879702.521704:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:1.0:1788879702.521707:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879702.521714:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879702.521718:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879702.521720:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879702.521721:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521724:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879702.521727:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521730:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305313801 00000100:00000001:1.0:1788879702.521732:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305314345 00000100:00000001:1.0:1788879702.521735:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305313801 00000100:00000001:1.0:1788879702.521737:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305314345 00000100:00000001:1.0:1788879702.521740:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879702.521742:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521744:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879702.521748:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879702/real 1788879702] req@ffff94e50449c380 x1875765290189952/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879702.521758:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879702.521759:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879702.521762:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879702.521763:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879702.521765:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521767:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879702.521769:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879702.521772:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290189952/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879702.521777:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521778:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879702.521780:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521781:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879702.521782:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521783:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879702.521795:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879702.521797:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879702.521799:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879702.521800:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879702.521803:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290189952:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879702.521806:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879702.521807:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879702.521808:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879702.521809:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879702.521810:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879702.521811:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879702.521813:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290190080:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879702.521815:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879702.521817:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879702.521818:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879702.521819:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879702.521821:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521824:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879702.521825:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:1.0:1788879702.521828:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879702.521836:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879702.521839:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879702.521841:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879702.521842:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521843:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879702.521844:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521847:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305313801 00000100:00000001:1.0:1788879702.521848:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305314345 00000100:00000001:1.0:1788879702.521850:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305313801 00000100:00000001:1.0:1788879702.521852:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305314345 00000100:00000001:1.0:1788879702.521853:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879702.521854:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521855:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879702.521857:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879702/real 1788879702] req@ffff94e50449f800 x1875765290190080/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879702.521867:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879702.521868:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879702.521869:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879702.521870:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879702.521871:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521872:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879702.521873:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879702.521876:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290190080/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879702.521880:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521881:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879702.521883:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521884:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879702.521885:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521886:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879702.521897:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879702.521898:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879702.521900:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879702.521923:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879702.521925:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290190080:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879702.521927:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521928:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879702.521929:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879702.521930:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879702.521932:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879702.521932:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879702.521933:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879702.521935:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879702.521936:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879702.521938:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879702.521939:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521940:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879702.521941:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879702.521942:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879702.521943:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879702.521944:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879702.521945:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879702.521946:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879702.521947:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879702.521948:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879702.521949:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521950:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879702.521950:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879702.521951:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879702.521952:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879702.521952:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879702.521953:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879702.521954:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879702.521955:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879702.521956:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879702.521956:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521958:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.521960:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879702.521961:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.521962:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879702.521963:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879702.585029:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879702.585033:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879702.585035:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879702.585039:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879702.585041:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879702.585044:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.585046:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879702.585048:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879702.585049:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879703.289072:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879703.289080:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879703.289087:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879703.289089:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879703.289115:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879703.289118:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879703.289122:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879703.608993:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879703.609012:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879703.609015:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879703.609020:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879703.609023:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879703.609027:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879703.609029:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879703.609033:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879703.609037:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879704.313039:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879704.313047:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879704.313053:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879704.313055:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879704.313088:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879704.313090:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879704.313093:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879704.633070:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879704.633079:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879704.633082:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879704.633089:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879704.633094:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879704.633100:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879704.633105:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879704.633110:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879704.633115:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879705.337087:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879705.337097:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879705.337107:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879705.337113:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879705.337117:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879705.337124:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879705.337131:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879705.337135:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879705.337139:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879705.337144:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879705.337319:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879705.337332:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879705.337343:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879705.657122:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879705.657135:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879705.657139:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879705.657148:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879705.657155:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879705.657162:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879705.657167:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879705.657172:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879705.657177:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879706.361000:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879706.361008:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879706.361013:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879706.361015:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879706.361141:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879706.361150:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879706.361156:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879706.681054:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879706.681062:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879706.681065:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879706.681071:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879706.681076:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879706.681081:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879706.681085:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879706.681088:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879706.681091:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879707.385156:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879707.385170:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879707.385176:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879707.385183:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879707.385191:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879707.385195:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879707.385199:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879707.385203:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879707.385210:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879707.385213:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879707.385268:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879707.385272:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879707.385276:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879707.641069:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879707.641083:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879707.641086:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879707.641091:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879707.641097:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879707.641106:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10646 00000100:00080000:3.0:1788879707.641121:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879707.641125:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879707.641127:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879707.641139:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879707.641142:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879707.641147:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879707.641150:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879707.641153:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879707.641154:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879707.641160:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879707.641163:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879707.641166:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879707.641172:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879707.641175:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879707.641178:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879707.641181:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879707.641210:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879707.641212:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879707.641217:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879707.641220:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879707.641226:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879707.641243:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879707.641255:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290190208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879707.641269:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879707.641275:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879707.641294:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879707.641299:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879707.641303:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879707.641305:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879707.641307:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879707.641310:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879707.641313:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10646 00000100:00080000:3.0:1788879707.641317:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879707.641319:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879707.641321:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879707.641325:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879707.641327:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879707.641329:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879707.641332:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879707.641336:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879707.641338:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879707.641342:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879707.641345:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879707.641348:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879707.641359:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879707.641360:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788879707.641363:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.641368:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879707.641369:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879707.641377:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1788879707.641380:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879707.641382:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:3.0:1788879707.641385:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.641388:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879707.641391:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879707.641398:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879707.641399:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879707.641400:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879707.641405:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879707.641406:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879707.641411:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879707.641415:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879707.641417:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879707.641425:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879707.641426:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290190208:192.168.203.129@tcp:250:kworker.0 00000100:00100000:3.0:1788879707.641438:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290190336/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879707.641440:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879707.641448:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879707.641454:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879707.641461:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879707.641465:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879707.641469:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879707.641472:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879707.641475:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788879707.641476:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879707.641481:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879707.641483:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1788879707.641485:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879707.641487:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879707.641495:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10646 00000100:00080000:3.0:1788879707.641499:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879707.641502:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879707.641504:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879707.641507:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879707.641508:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879707.641511:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879707.641515:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1788879707.641515:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879707.641521:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:3.0:1788879707.641523:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879707.641528:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879707.641532:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879707.641539:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879707.641546:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879707.641550:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879707.641555:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879707.641560:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879707.641563:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879707.641567:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879707.641572:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879707.641575:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879707.641584:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879707.641589:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879707.641591:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879707.641599:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879707.641605:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879707.641610:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.641614:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879707.641623:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.641624:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879707.641629:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.641634:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:3.0:1788879707.641640:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290190464/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879707.641643:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.641655:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305319465 00000100:00000001:3.0:1788879707.641661:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879707.641666:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.641673:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879707.641674:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879707.641683:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879707.641688:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879707.641692:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879707.641694:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879707.641698:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305319465 00000100:00000001:3.0:1788879707.641699:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879707.641708:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10646 00000100:00000001:1.0:1788879707.641711:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788879707.641715:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879707.641719:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879707.641721:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879707.641726:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879707.641728:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879707.641733:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879707.641735:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1788879707.641739:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879707.641744:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:1.0:1788879707.641748:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879707/real 1788879707] req@ffff94e50449f480 x1875765290190208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788879707.641749:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879707.641759:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879707.641764:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:1.0:1788879707.641780:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1788879707.641780:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000400:00000001:1.0:1788879707.641787:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879707.641789:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879707.641794:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879707.641796:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:3.0:1788879707.641799:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879707.641802:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:3.0:1788879707.641807:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.641808:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879707.641812:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879707.641815:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00100000:3.0:1788879707.641818:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879707.641819:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1788879707.641825:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879707.641832:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879707.641833:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290190208/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879707.641839:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879707.641844:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879707.641850:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879707.641853:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879707.641857:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879707.641858:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.641867:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879707.641869:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.641876:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.641882:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:3.0:1788879707.641886:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290190592/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879707.641887:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.641894:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879707.641899:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879707.641952:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879707.641957:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879707.641964:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879707.641995:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879707.642007:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879707.642013:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879707.642021:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290190208:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879707.642031:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642035:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879707.642037:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879707.642040:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879707.642046:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879707.642049:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879707.642051:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879707.642056:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879707.642058:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879707.642062:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879707.642064:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642067:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642076:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879707.642078:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.642081:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879707.642083:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879707.642085:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879707.642087:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879707.642089:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879707.642092:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879707.642094:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879707.642097:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879707.642101:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290190336:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879707.642105:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879707.642108:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879707.642110:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879707.642114:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879707.642119:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.642131:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879707.642134:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:1.0:1788879707.642142:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879707.642168:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879707.642193:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879707.642198:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879707.642201:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.642204:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879707.642209:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.642217:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305318924 00000100:00000001:1.0:1788879707.642221:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305319465 00000100:00000001:1.0:1788879707.642224:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305318924 00000100:00000001:1.0:1788879707.642226:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305319465 00000100:00000001:1.0:1788879707.642229:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879707.642232:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.642234:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879707.642240:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879707/real 1788879707] req@ffff94e50449c700 x1875765290190336/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879707.642252:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879707.642253:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879707.642257:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879707.642259:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879707.642261:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642263:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879707.642265:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879707.642269:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290190336/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879707.642278:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642280:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879707.642284:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642286:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879707.642287:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642289:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879707.642305:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879707.642308:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879707.642312:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879707.642315:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879707.642319:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290190336:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879707.642323:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879707.642324:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879707.642326:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879707.642328:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879707.642331:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879707.642333:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879707.642337:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290190464:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879707.642341:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879707.642343:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879707.642345:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879707.642347:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879707.642351:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.642358:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879707.642359:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:1.0:1788879707.642364:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879707.642379:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879707.642384:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879707.642386:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879707.642388:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.642390:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879707.642393:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.642397:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305318922 00000100:00000001:1.0:1788879707.642400:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305319465 00000100:00000001:1.0:1788879707.642403:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305318922 00000100:00000001:1.0:1788879707.642406:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305319465 00000100:00000001:1.0:1788879707.642408:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879707.642411:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.642413:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879707.642417:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879707/real 1788879707] req@ffff94e50449e300 x1875765290190464/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879707.642426:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879707.642427:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879707.642430:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879707.642432:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879707.642434:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642436:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879707.642438:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879707.642442:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290190464/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879707.642449:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642451:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879707.642454:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642456:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879707.642457:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642459:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879707.642472:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879707.642474:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879707.642477:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879707.642480:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879707.642483:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290190464:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879707.642487:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879707.642488:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879707.642490:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879707.642493:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879707.642495:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879707.642497:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879707.642500:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290190592:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879707.642504:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879707.642505:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879707.642507:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879707.642509:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879707.642512:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.642516:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879707.642517:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:1.0:1788879707.642521:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879707.642531:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879707.642535:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879707.642537:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879707.642539:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.642541:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879707.642543:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.642547:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305318922 00000100:00000001:1.0:1788879707.642550:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305319465 00000100:00000001:1.0:1788879707.642553:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305318922 00000100:00000001:1.0:1788879707.642555:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305319465 00000100:00000001:1.0:1788879707.642558:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879707.642560:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.642562:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879707.642566:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879707/real 1788879707] req@ffff94e50449c000 x1875765290190592/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879707.642574:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879707.642575:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879707.642577:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879707.642579:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879707.642580:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642583:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879707.642585:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879707.642589:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290190592/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879707.642597:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642599:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879707.642602:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642603:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879707.642605:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642606:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879707.642619:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879707.642622:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879707.642625:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879707.642627:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879707.642630:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290190592:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879707.642634:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642636:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879707.642638:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879707.642639:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879707.642642:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879707.642644:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879707.642645:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879707.642648:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879707.642650:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879707.642652:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879707.642654:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642656:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879707.642657:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879707.642659:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879707.642661:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879707.642663:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879707.642664:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879707.642666:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879707.642667:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879707.642670:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879707.642672:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642674:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879707.642675:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879707.642676:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879707.642678:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879707.642679:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879707.642681:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879707.642682:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879707.642683:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879707.642685:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879707.642686:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642688:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.642692:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879707.642693:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.642696:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879707.642697:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879707.705061:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879707.705073:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879707.705076:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879707.705086:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879707.705091:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879707.705096:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.705100:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879707.705103:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879707.705106:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879708.409121:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879708.409148:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879708.409158:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879708.409176:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879708.409205:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879708.409208:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879708.409211:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879708.729085:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879708.729113:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879708.729116:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879708.729124:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879708.729130:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879708.729136:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879708.729140:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879708.729145:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879708.729150:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879709.433078:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879709.433086:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879709.433110:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879709.433111:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879709.433179:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879709.433185:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879709.433190:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879709.753053:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879709.753063:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879709.753066:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879709.753073:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879709.753078:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879709.753084:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879709.753089:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879709.753093:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879709.753097:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879710.457069:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879710.457079:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879710.457085:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879710.457087:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879710.457147:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879710.457150:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879710.457153:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879710.777077:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879710.777086:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879710.777088:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879710.777095:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879710.777099:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879710.777103:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879710.777106:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879710.777109:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879710.777111:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879711.481080:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879711.481087:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879711.481094:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879711.481096:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879711.481127:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879711.481130:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879711.481133:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879711.801044:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879711.801052:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879711.801055:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879711.801069:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879711.801076:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879711.801081:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879711.801084:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879711.801087:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879711.801091:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879712.505010:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879712.505017:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879712.505025:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879712.505027:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879712.505051:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879712.505055:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879712.505058:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879712.761136:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879712.761151:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879712.761156:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879712.761162:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879712.761169:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879712.761180:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10651 00000100:00080000:0.0:1788879712.761193:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879712.761196:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879712.761199:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879712.761207:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879712.761210:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879712.761213:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879712.761216:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879712.761218:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879712.761220:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879712.761225:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879712.761227:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879712.761231:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879712.761235:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879712.761238:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879712.761241:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879712.761244:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879712.761257:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879712.761259:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879712.761263:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879712.761266:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879712.761271:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879712.761299:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879712.761309:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978e00 x1875765290190720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879712.761320:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879712.761324:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879712.761339:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879712.761342:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879712.761346:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879712.761347:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879712.761350:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879712.761352:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879712.761355:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10651 00000100:00080000:0.0:1788879712.761358:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879712.761360:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879712.761361:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879712.761364:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879712.761365:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879712.761367:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:0.0:1788879712.761369:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879712.761371:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879712.761372:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879712.761376:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879712.761378:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879712.761379:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879712.761400:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879712.761403:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879712.761406:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879712.761410:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879712.761413:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879712.761415:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.761419:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1788879712.761420:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879712.761424:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879712.761427:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879712.761428:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1788879712.761429:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879712.761435:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.761435:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879712.761442:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1788879712.761444:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.761445:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879712.761455:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:0.0:1788879712.761457:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897a300 x1875765290190848/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879712.761460:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879712.761469:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879712.761476:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879712.761481:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879712.761482:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290190720:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879712.761485:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.761492:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1788879712.761493:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879712.761500:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:1.0:1788879712.761500:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879712.761505:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879712.761507:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1788879712.761512:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1788879712.761514:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879712.761515:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879712.761523:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879712.761528:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10651 00000100:00080000:0.0:1788879712.761531:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879712.761533:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879712.761534:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879712.761538:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1788879712.761539:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879712.761544:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145412608 : -117763564139008 : ffff94e508978e00) 00000020:00000001:0.0:1788879712.761545:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879712.761550:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:0.0:1788879712.761554:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879712.761554:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879712.761576:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879712.761579:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879712.761585:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879712.761591:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879712.761594:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879712.761599:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879712.761602:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879712.761606:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879712.761610:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879712.761615:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879712.761621:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879712.761622:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:0.0:1788879712.761634:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879712.761639:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:0.0:1788879712.761640:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879712.761648:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879712.761649:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1788879712.761653:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879712.761655:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879712.761661:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.761661:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879712.761670:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879712.761673:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.761677:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305324585 00000100:00100000:0.0:1788879712.761685:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979c00 x1875765290190976/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879712.761690:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1788879712.761699:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879712.761703:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.761704:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305324585 00000100:00000001:0.0:1788879712.761708:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.761713:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:0.0:1788879712.761714:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879712.761723:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1788879712.761724:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879712.761729:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879712.761732:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879712.761733:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879712.761739:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879712.761741:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879712.761747:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10651 00000100:00100000:1.0:1788879712.761754:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879712/real 1788879712] req@ffff94e508978e00 x1875765290190720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1788879712.761756:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879712.761761:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879712.761764:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879712.761772:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879712.761777:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879712.761782:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:0.0:1788879712.761789:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000400:00000001:1.0:1788879712.761791:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788879712.761797:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000400:00000001:1.0:1788879712.761798:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1788879712.761803:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879712.761809:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:0.0:1788879712.761817:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879712.761818:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879712.761825:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879712.761827:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.761834:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1788879712.761835:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879712.761839:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879712.761848:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978e00 x1875765290190720/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1788879712.761856:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879712.761859:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1788879712.761862:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.761866:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1788879712.761868:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879712.761873:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879712.761875:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1788879712.761880:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.761881:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879712.761886:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879712.761888:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.761892:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879712.761896:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879712.761900:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879712.761906:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879712.761909:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879712.761914:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290190720:192.168.203.129@tcp:250:kworker.0 02000000:00000001:0.0:1788879712.761980:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879712.762000:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879712.762002:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879712.762004:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879712.762006:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879712.762013:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879712.762020:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897bb80 x1875765290191104/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879712.762028:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879712.762030:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879712.762041:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.762086:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762089:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879712.762093:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879712.762106:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879712.762110:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879712.762113:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879712.762115:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879712.762119:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879712.762122:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879712.762126:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879712.762128:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762130:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762146:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879712.762150:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.762155:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879712.762157:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879712.762160:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879712.762163:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879712.762165:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879712.762169:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879712.762172:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879712.762175:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879712.762181:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290190848:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879712.762187:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879712.762190:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879712.762193:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879712.762196:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879712.762201:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.762211:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879712.762214:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145417984 : -117763564133632 : ffff94e50897a300) 00000100:00000001:1.0:1788879712.762220:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879712.762240:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879712.762245:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879712.762249:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879712.762250:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.762252:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879712.762255:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.762278:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305324044 00000100:00000001:1.0:1788879712.762283:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305324585 00000100:00000001:1.0:1788879712.762286:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305324044 00000100:00000001:1.0:1788879712.762290:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305324585 00000100:00000001:1.0:1788879712.762293:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879712.762296:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.762299:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879712.762306:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879712/real 1788879712] req@ffff94e50897a300 x1875765290190848/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879712.762319:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879712.762321:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879712.762326:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879712.762328:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879712.762330:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762333:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879712.762335:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879712.762341:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897a300 x1875765290190848/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879712.762352:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762356:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879712.762360:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762362:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879712.762364:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762366:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879712.762387:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879712.762402:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879712.762408:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879712.762414:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879712.762419:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290190848:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879712.762423:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879712.762425:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879712.762426:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879712.762429:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879712.762431:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879712.762433:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879712.762435:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290190976:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879712.762439:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879712.762441:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879712.762443:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879712.762444:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879712.762447:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.762454:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879712.762455:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145416192 : -117763564135424 : ffff94e508979c00) 00000100:00000001:1.0:1788879712.762459:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879712.762472:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879712.762477:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879712.762479:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879712.762480:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.762482:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879712.762484:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.762487:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305324042 00000100:00000001:1.0:1788879712.762490:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305324585 00000100:00000001:1.0:1788879712.762492:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305324042 00000100:00000001:1.0:1788879712.762494:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305324585 00000100:00000001:1.0:1788879712.762497:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879712.762498:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.762500:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879712.762503:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879712/real 1788879712] req@ffff94e508979c00 x1875765290190976/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879712.762518:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879712.762519:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879712.762521:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879712.762523:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879712.762524:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762526:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879712.762527:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879712.762530:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979c00 x1875765290190976/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879712.762537:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762539:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879712.762541:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762543:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879712.762544:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762546:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879712.762571:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879712.762590:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879712.762593:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879712.762595:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879712.762597:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290190976:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879712.762601:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879712.762602:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879712.762603:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879712.762605:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879712.762607:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879712.762609:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879712.762611:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290191104:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879712.762615:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879712.762617:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879712.762619:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879712.762621:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879712.762623:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.762627:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879712.762629:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145424256 : -117763564127360 : ffff94e50897bb80) 00000100:00000001:1.0:1788879712.762633:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879712.762643:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879712.762646:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879712.762657:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879712.762659:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.762660:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879712.762662:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.762666:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305324042 00000100:00000001:1.0:1788879712.762668:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305324585 00000100:00000001:1.0:1788879712.762670:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305324042 00000100:00000001:1.0:1788879712.762673:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305324585 00000100:00000001:1.0:1788879712.762675:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879712.762677:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.762678:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879712.762682:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879712/real 1788879712] req@ffff94e50897bb80 x1875765290191104/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879712.762689:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879712.762690:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879712.762691:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879712.762693:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879712.762694:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762695:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879712.762697:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879712.762700:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897bb80 x1875765290191104/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879712.762706:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762709:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879712.762711:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762713:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879712.762714:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762716:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879712.762731:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879712.762734:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879712.762737:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879712.762739:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879712.762742:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290191104:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879712.762745:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762747:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879712.762748:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879712.762749:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879712.762752:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879712.762754:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879712.762756:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879712.762759:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879712.762761:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879712.762765:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879712.762767:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762769:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879712.762771:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879712.762773:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879712.762775:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879712.762777:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879712.762779:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879712.762782:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879712.762783:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879712.762786:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879712.762788:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762791:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879712.762793:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879712.762795:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879712.762797:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879712.762799:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879712.762801:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879712.762804:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879712.762807:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879712.762810:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879712.762812:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762815:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.762821:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879712.762823:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.762826:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879712.762828:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879712.825031:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879712.825036:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879712.825038:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879712.825043:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879712.825046:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879712.825050:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.825053:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879712.825055:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879712.825057:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879713.529044:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879713.529050:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879713.529056:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879713.529057:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879713.529126:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879713.529131:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879713.529135:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879713.849000:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879713.849008:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879713.849011:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879713.849016:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879713.849019:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879713.849022:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879713.849024:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879713.849026:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879713.849028:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879714.553055:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879714.553064:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879714.553069:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879714.553071:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879714.553165:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879714.553181:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879714.553185:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879714.873048:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879714.873058:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879714.873061:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879714.873068:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879714.873072:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879714.873077:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879714.873080:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879714.873083:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879714.873087:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879715.577069:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879715.577079:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879715.577087:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879715.577092:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879715.577095:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879715.577101:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879715.577104:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879715.577107:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879715.577109:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879715.577112:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879715.577142:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879715.577147:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879715.577152:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879715.897059:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879715.897068:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879715.897070:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879715.897076:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879715.897081:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879715.897085:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879715.897089:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879715.897092:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879715.897094:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879716.600988:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879716.600998:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879716.601003:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879716.601005:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879716.601027:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879716.601031:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879716.601034:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879716.920994:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879716.921013:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879716.921014:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879716.921020:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879716.921024:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879716.921027:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879716.921030:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879716.921032:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879716.921035:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879717.625035:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879717.625045:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879717.625048:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879717.625052:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879717.625058:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879717.625060:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879717.625062:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879717.625065:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879717.625068:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879717.625069:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879717.625170:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879717.625179:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879717.625185:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879717.881059:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879717.881067:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879717.881070:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879717.881073:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879717.881077:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879717.881083:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10656 00000100:00080000:1.0:1788879717.881093:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879717.881095:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879717.881097:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879717.881104:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879717.881106:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879717.881109:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879717.881111:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879717.881113:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879717.881114:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879717.881117:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879717.881119:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879717.881121:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879717.881124:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879717.881127:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881129:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879717.881131:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879717.881141:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879717.881142:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881144:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879717.881147:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879717.881151:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881160:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879717.881168:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100bb80 x1875765290191232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879717.881176:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879717.881180:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881197:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879717.881199:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879717.881202:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879717.881203:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879717.881204:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879717.881207:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879717.881208:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10656 00000100:00080000:1.0:1788879717.881210:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879717.881211:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879717.881212:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879717.881214:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879717.881215:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879717.881217:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879717.881218:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879717.881220:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879717.881221:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879717.881222:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879717.881229:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879717.881231:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879717.881236:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879717.881238:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881239:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879717.881242:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879717.881245:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881247:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879717.881252:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879717.881254:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881255:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879717.881256:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879717.881259:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881262:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.881266:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1788879717.881267:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008700 x1875765290191360/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879717.881276:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879717.881280:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879717.881281:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879717.881285:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.881286:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879717.881290:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881292:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879717.881302:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1788879717.881303:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1788879717.881313:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:2.0:1788879717.881313:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879717.881320:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1788879717.881321:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879717.881327:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879717.881331:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00100000:2.0:1788879717.881334:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290191232:192.168.203.129@tcp:250:kworker.0 00000100:00080000:1.0:1788879717.881335:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10656 00000100:00080000:1.0:1788879717.881349:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879717.881350:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788879717.881352:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879717.881355:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879717.881356:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1788879717.881361:0:178134:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1788879717.881361:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879717.881366:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:2.0:1788879717.881368:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:1.0:1788879717.881370:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879717.881374:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 02000000:00000001:2.0:1788879717.881375:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881376:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879717.881383:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879717.881400:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879717.881402:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879717.881404:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879717.881406:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879717.881408:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881409:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879717.881411:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879717.881413:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881414:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879717.881416:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879717.881417:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.881418:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879717.881419:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879717.881421:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286523264 : -117763423028352 : ffff94e51100bb80) 02000000:00000001:1.0:1788879717.881422:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879717.881430:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.881431:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879717.881441:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879717.881445:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008e00 x1875765290191488/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879717.881451:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879717.881453:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881455:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879717.881457:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879717.881459:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879717.881460:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879717.881461:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879717.881463:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879717.881465:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10656 00000100:00080000:1.0:1788879717.881469:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879717.881470:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879717.881471:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879717.881474:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879717.881476:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879717.881477:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:1.0:1788879717.881484:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000800:00000100:2.0:1788879717.881484:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879717.881491:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879717.881493:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879717.881497:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879717.881499:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879717.881500:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879717.881504:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:1.0:1788879717.881505:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879717.881512:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881515:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879717.881516:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000008:00100000:1.0:1788879717.881517:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879717.881521:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.881521:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881536:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.881536:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:1.0:1788879717.881544:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879717.881545:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879717.881547:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881552:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879717.881553:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305329705 02000000:00000001:1.0:1788879717.881554:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879717.881559:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881564:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.881569:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00100000:1.0:1788879717.881570:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008000 x1875765290191616/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879717.881581:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879717.881582:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.881584:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.881589:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305329705 00000100:00000001:2.0:1788879717.881598:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879717.881603:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879717.881605:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.881609:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879717.881618:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879717/real 1788879717] req@ffff94e51100bb80 x1875765290191232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879717.881629:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879717.881631:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879717.881635:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879717.881637:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879717.881639:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.881641:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879717.881642:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879717.881646:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100bb80 x1875765290191232/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879717.881652:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.881655:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879717.881657:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.881658:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879717.881659:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.881662:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879717.881664:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879717.881667:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879717.881673:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879717.881676:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879717.881681:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290191232:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879717.881687:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.881690:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879717.881692:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879717.881695:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879717.881699:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879717.881701:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879717.881704:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879717.881710:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879717.881712:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879717.881717:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879717.881718:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.881721:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.881730:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879717.881732:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.881735:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879717.881738:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879717.881739:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879717.881741:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879717.881742:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879717.881745:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879717.881746:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879717.881748:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879717.881751:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290191360:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879717.881754:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879717.881756:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879717.881758:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879717.881759:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879717.881778:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.881784:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879717.881785:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286509824 : -117763423041792 : ffff94e511008700) 00000100:00000001:2.0:1788879717.881790:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879717.881801:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879717.881805:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879717.881806:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879717.881808:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.881809:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879717.881811:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.881815:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305329163 00000100:00000001:2.0:1788879717.881817:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305329705 00000100:00000001:2.0:1788879717.881820:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305329163 00000100:00000001:2.0:1788879717.881822:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305329705 00000100:00000001:2.0:1788879717.881824:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879717.881825:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.881827:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879717.881829:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879717/real 1788879717] req@ffff94e511008700 x1875765290191360/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879717.881835:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879717.881836:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879717.881838:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879717.881839:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879717.881840:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.881842:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879717.881844:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879717.881848:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008700 x1875765290191360/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879717.881856:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.881858:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879717.881861:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.881863:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879717.881865:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.881867:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879717.881898:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879717.881963:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879717.881966:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879717.881968:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879717.881971:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290191360:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879717.881974:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879717.881976:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879717.881977:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879717.881979:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879717.881981:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879717.881982:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879717.881984:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290191488:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879717.881987:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879717.881988:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879717.881990:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879717.881991:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879717.881994:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.882000:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879717.882001:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286511616 : -117763423040000 : ffff94e511008e00) 00000100:00000001:2.0:1788879717.882004:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879717.882013:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879717.882017:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879717.882019:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879717.882020:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.882021:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879717.882023:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.882026:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305329162 00000100:00000001:2.0:1788879717.882028:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305329705 00000100:00000001:2.0:1788879717.882030:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305329162 00000100:00000001:2.0:1788879717.882031:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305329705 00000100:00000001:2.0:1788879717.882033:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879717.882034:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.882035:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879717.882038:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879717/real 1788879717] req@ffff94e511008e00 x1875765290191488/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879717.882043:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879717.882044:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879717.882045:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879717.882047:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879717.882048:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.882049:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879717.882051:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879717.882053:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008e00 x1875765290191488/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879717.882058:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.882059:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879717.882061:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.882062:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879717.882063:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.882064:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879717.882074:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879717.882075:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879717.882077:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879717.882079:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879717.882081:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290191488:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879717.882083:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879717.882084:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879717.882085:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879717.882087:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879717.882088:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879717.882089:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879717.882092:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290191616:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879717.882094:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879717.882095:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879717.882096:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879717.882097:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879717.882099:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.882102:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879717.882102:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508032 : -117763423043584 : ffff94e511008000) 00000100:00000001:2.0:1788879717.882105:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879717.882111:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879717.882114:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879717.882115:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879717.882116:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.882118:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879717.882119:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.882122:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305329162 00000100:00000001:2.0:1788879717.882124:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305329705 00000100:00000001:2.0:1788879717.882125:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305329162 00000100:00000001:2.0:1788879717.882127:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305329705 00000100:00000001:2.0:1788879717.882129:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879717.882130:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.882131:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879717.882134:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879717/real 1788879717] req@ffff94e511008000 x1875765290191616/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879717.882138:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879717.882139:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879717.882141:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879717.882142:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879717.882142:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.882144:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879717.882144:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879717.882147:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008000 x1875765290191616/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879717.882151:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.882153:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879717.882155:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.882156:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879717.882157:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.882158:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879717.882168:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879717.882169:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879717.882172:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879717.882174:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879717.882177:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290191616:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879717.882179:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.882180:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879717.882181:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879717.882182:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879717.882184:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879717.882185:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879717.882186:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879717.882188:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879717.882188:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879717.882190:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879717.882191:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.882192:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879717.882193:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879717.882194:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879717.882195:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879717.882196:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879717.882197:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879717.882198:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879717.882199:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879717.882200:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879717.882201:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.882202:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879717.882202:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879717.882203:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879717.882205:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879717.882206:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879717.882207:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879717.882208:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879717.882208:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879717.882209:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879717.882210:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.882211:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879717.882213:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879717.882214:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879717.882216:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879717.882217:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879717.944976:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879717.944979:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879717.944981:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879717.944985:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879717.944987:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879717.944990:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879717.944992:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879717.944993:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879717.944995:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879718.649003:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879718.649010:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879718.649015:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879718.649016:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879718.649067:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879718.649071:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879718.649073:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879718.969003:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879718.969008:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879718.969010:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879718.969015:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879718.969018:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879718.969021:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879718.969024:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879718.969026:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879718.969028:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879719.673062:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879719.673069:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879719.673074:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879719.673075:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879719.673098:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879719.673101:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879719.673104:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879719.993074:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879719.993083:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879719.993086:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879719.993093:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879719.993098:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879719.993114:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879719.993118:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879719.993121:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879719.993125:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879720.697032:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879720.697040:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879720.697047:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879720.697049:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879720.697073:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879720.697076:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879720.697080:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879721.017063:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879721.017073:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879721.017076:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879721.017082:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879721.017086:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879721.017091:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879721.017094:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879721.017097:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879721.017101:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879721.721034:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879721.721039:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879721.721044:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879721.721045:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879721.721128:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879721.721136:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879721.721142:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879722.040997:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879722.041003:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879722.041006:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879722.041010:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879722.041012:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879722.041017:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879722.041019:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879722.041022:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879722.041023:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879722.745012:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879722.745017:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879722.745021:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879722.745022:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879722.745087:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879722.745092:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879722.745096:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879723.001014:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879723.001024:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879723.001028:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879723.001033:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879723.001037:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879723.001044:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10661 00000100:00080000:1.0:1788879723.001054:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879723.001057:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879723.001060:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879723.001068:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879723.001070:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879723.001074:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879723.001077:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879723.001080:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879723.001081:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879723.001086:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879723.001090:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879723.001093:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879723.001097:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879723.001100:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879723.001102:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879723.001105:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879723.001117:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879723.001119:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879723.001123:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879723.001125:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879723.001130:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879723.001142:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879723.001151:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100ad80 x1875765290191744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879723.001163:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879723.001167:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879723.001184:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879723.001188:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879723.001194:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879723.001196:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879723.001198:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879723.001201:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879723.001204:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10661 00000100:00080000:1.0:1788879723.001208:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879723.001210:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879723.001211:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879723.001214:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879723.001216:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879723.001218:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879723.001221:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879723.001223:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879723.001225:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879723.001229:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879723.001233:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879723.001235:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879723.001241:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879723.001244:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879723.001247:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879723.001251:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879723.001255:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879723.001258:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.001262:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1788879723.001263:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879723.001274:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.001274:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879723.001282:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879723.001284:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879723.001288:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788879723.001289:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879723.001296:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879723.001297:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879723.001303:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788879723.001307:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879723.001307:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879723.001311:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879723.001319:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008000 x1875765290191872/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1788879723.001321:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290191744:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879723.001336:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879723.001344:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879723.001348:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879723.001353:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788879723.001363:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788879723.001363:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879723.001371:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879723.001379:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879723.001382:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879723.001385:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879723.001389:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286519680 : -117763423031936 : ffff94e51100ad80) 00000100:00080000:1.0:1788879723.001390:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879723.001399:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879723.001400:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1788879723.001406:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879723.001411:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879723.001415:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10661 00000100:00080000:1.0:1788879723.001421:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879723.001424:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879723.001426:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879723.001429:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879723.001432:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879723.001435:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879723.001438:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879723.001440:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879723.001443:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879723.001448:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:2.0:1788879723.001450:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879723.001461:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879723.001465:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879723.001470:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879723.001473:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.001474:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879723.001480:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879723.001481:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879723.001487:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1788879723.001489:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879723.001492:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000008:00000001:1.0:1788879723.001498:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.001500:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879723.001502:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.001506:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305334825 02000000:00000001:1.0:1788879723.001509:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879723.001512:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879723.001516:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879723.001518:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879723.001518:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1788879723.001521:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879723.001530:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879723.001536:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008e00 x1875765290192000/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879723.001537:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305334825 00000100:00000001:2.0:1788879723.001546:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879723.001553:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879723.001560:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879723.001562:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879723.001564:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1788879723.001568:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879723.001573:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879723/real 1788879723] req@ffff94e51100ad80 x1875765290191744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879723.001576:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879723.001586:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:2.0:1788879723.001593:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:1.0:1788879723.001594:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000400:00000001:2.0:1788879723.001596:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879723.001599:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879723.001601:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:1.0:1788879723.001602:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879723.001605:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879723.001608:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879723.001610:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.001615:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:1.0:1788879723.001617:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10661 00000100:00000001:2.0:1788879723.001619:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1788879723.001629:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1788879723.001631:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100ad80 x1875765290191744/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879723.001637:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879723.001641:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879723.001654:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1788879723.001655:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879723.001660:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:1.0:1788879723.001661:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879723.001668:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1788879723.001669:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879723.001673:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:1.0:1788879723.001679:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879723.001679:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879723.001690:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879723.001690:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879723.001693:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879723.001695:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879723.001701:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879723.001708:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00080000:2.0:1788879723.001708:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879723.001712:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1788879723.001719:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879723.001722:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290191744:192.168.203.129@tcp:250:kworker.0 02000000:00000001:1.0:1788879723.001727:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879723.001729:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.001733:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879723.001735:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879723.001736:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879723.001737:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879723.001742:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879723.001743:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879723.001747:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:1.0:1788879723.001750:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000020:00000001:2.0:1788879723.001750:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000008:00100000:1.0:1788879723.001760:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1788879723.001762:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879723.001767:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000008:00000001:1.0:1788879723.001769:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.001771:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879723.001775:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879723.001777:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.001779:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788879723.001786:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879723.001788:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:1.0:1788879723.001790:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.001791:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.001799:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879723.001800:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879723.001803:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1788879723.001805:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879723.001806:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879723.001810:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788879723.001811:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.001812:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879723.001820:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879723.001822:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879723.001826:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879723.001826:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879723.001830:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290191872:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879723.001834:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879723.001837:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879723.001840:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879723.001842:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008700 x1875765290192128/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788879723.001844:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879723.001848:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.001858:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879723.001860:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508032 : -117763423043584 : ffff94e511008000) 00000100:00000001:1.0:1788879723.001876:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879723.001882:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.001883:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879723.001893:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788879723.001901:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879723.001930:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879723.001932:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879723.001934:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.001937:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879723.001939:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.001943:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305334284 00000100:00000001:2.0:1788879723.001946:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305334825 00000100:00000001:2.0:1788879723.001949:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305334284 00000100:00000001:2.0:1788879723.001952:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305334825 00000100:00000001:2.0:1788879723.001955:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879723.001958:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.001959:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879723.001962:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879723/real 1788879723] req@ffff94e511008000 x1875765290191872/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879723.001969:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879723.001970:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879723.001971:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879723.001973:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879723.001973:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.001974:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879723.001976:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879723.001978:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008000 x1875765290191872/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879723.001982:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.001984:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879723.001985:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.001986:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879723.001987:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.001988:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879723.001999:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879723.002001:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879723.002004:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879723.002005:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879723.002007:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290191872:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879723.002010:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879723.002011:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879723.002012:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879723.002013:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879723.002015:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879723.002016:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879723.002018:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290192000:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879723.002021:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879723.002022:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879723.002023:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879723.002024:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879723.002027:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.002031:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879723.002032:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286511616 : -117763423040000 : ffff94e511008e00) 00000100:00000001:2.0:1788879723.002034:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879723.002042:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879723.002044:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879723.002046:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879723.002047:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.002048:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879723.002049:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.002052:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305334282 00000100:00000001:2.0:1788879723.002054:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305334825 00000100:00000001:2.0:1788879723.002055:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305334282 00000100:00000001:2.0:1788879723.002057:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305334825 00000100:00000001:2.0:1788879723.002059:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879723.002060:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.002061:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879723.002064:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879723/real 1788879723] req@ffff94e511008e00 x1875765290192000/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879723.002068:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879723.002069:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879723.002070:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879723.002071:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879723.002072:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.002073:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879723.002074:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879723.002076:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008e00 x1875765290192000/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879723.002085:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.002087:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879723.002089:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.002090:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879723.002090:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.002091:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879723.002099:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879723.002100:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879723.002102:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879723.002103:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879723.002105:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290192000:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879723.002108:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.002109:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879723.002110:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879723.002111:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879723.002113:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879723.002114:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879723.002115:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879723.002117:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879723.002118:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879723.002119:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879723.002120:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.002121:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879723.002122:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879723.002123:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879723.002124:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879723.002125:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879723.002126:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879723.002127:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879723.002128:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879723.002129:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879723.002130:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.002131:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.002132:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879723.002134:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.002135:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879723.002136:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879723.002137:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879723.002138:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879723.002139:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879723.002141:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879723.002141:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879723.002143:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879723.002144:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290192128:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879723.002146:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879723.002148:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879723.002149:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879723.002150:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879723.002151:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.002154:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879723.002155:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286509824 : -117763423041792 : ffff94e511008700) 00000100:00000001:2.0:1788879723.002158:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879723.002164:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879723.002166:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879723.002167:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879723.002168:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.002169:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879723.002171:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.002173:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305334282 00000100:00000001:2.0:1788879723.002175:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305334825 00000100:00000001:2.0:1788879723.002176:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305334282 00000100:00000001:2.0:1788879723.002178:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305334825 00000100:00000001:2.0:1788879723.002179:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879723.002180:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.002181:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879723.002183:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879723/real 1788879723] req@ffff94e511008700 x1875765290192128/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879723.002192:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879723.002193:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879723.002195:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879723.002196:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879723.002197:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.002198:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879723.002198:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879723.002201:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008700 x1875765290192128/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879723.002205:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.002206:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879723.002208:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.002209:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879723.002210:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.002211:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879723.002220:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879723.002221:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879723.002223:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879723.002224:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879723.002226:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290192128:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879723.002229:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.002230:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879723.002231:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879723.002231:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879723.002232:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879723.002233:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879723.002234:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879723.002235:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879723.002236:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879723.002236:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879723.002237:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.002239:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879723.002240:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879723.002241:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879723.002242:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879723.002243:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879723.065015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879723.065022:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879723.065025:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879723.065031:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879723.065035:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879723.065039:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879723.065042:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879723.065045:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879723.065048:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879723.769015:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879723.769021:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879723.769024:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879723.769025:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879723.769043:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879723.769045:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879723.769048:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879724.089008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879724.089017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879724.089019:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879724.089025:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879724.089028:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879724.089031:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879724.089034:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879724.089036:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879724.089038:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879724.793054:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879724.793060:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879724.793064:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879724.793065:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879724.793082:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879724.793084:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879724.793087:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879725.112993:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879725.112998:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879725.113000:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879725.113004:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879725.113007:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879725.113011:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879725.113013:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879725.113015:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879725.113017:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879725.817045:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879725.817051:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879725.817066:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879725.817070:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879725.817073:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879725.817077:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879725.817080:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879725.817082:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879725.817085:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879725.817087:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879725.817180:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879725.817187:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879725.817192:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879726.136998:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879726.137005:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879726.137006:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879726.137010:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879726.137013:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879726.137017:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879726.137019:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879726.137022:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879726.137024:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879726.841029:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879726.841034:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879726.841038:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879726.841039:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879726.841100:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879726.841106:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879726.841112:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879727.161005:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879727.161013:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879727.161016:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879727.161021:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879727.161025:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879727.161030:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879727.161033:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879727.161036:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879727.161039:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879727.865018:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879727.865025:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879727.865033:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879727.865036:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879727.865040:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879727.865042:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879727.865043:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879727.865045:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879727.865048:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879727.865049:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879727.865067:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879727.865070:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879727.865072:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879728.121021:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879728.121029:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879728.121031:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879728.121035:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879728.121039:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879728.121045:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10666 00000100:00080000:3.0:1788879728.121054:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879728.121057:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879728.121059:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879728.121066:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879728.121068:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879728.121071:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879728.121073:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879728.121074:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879728.121075:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879728.121078:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879728.121080:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879728.121082:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879728.121085:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879728.121087:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879728.121089:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879728.121090:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879728.121100:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879728.121101:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879728.121104:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879728.121106:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879728.121110:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879728.121120:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879728.121127:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290192256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879728.121135:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879728.121138:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879728.121149:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879728.121152:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879728.121155:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879728.121156:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879728.121157:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879728.121159:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879728.121161:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10666 00000100:00080000:3.0:1788879728.121164:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879728.121165:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879728.121166:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879728.121169:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879728.121171:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879728.121173:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879728.121175:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879728.121177:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879728.121179:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879728.121182:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879728.121184:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879728.121185:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879728.121189:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879728.121191:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879728.121192:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879728.121195:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879728.121196:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:3.0:1788879728.121198:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879728.121205:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121206:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1788879728.121209:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879728.121211:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121212:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879728.121214:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879728.121217:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879728.121218:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788879728.121220:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121224:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879728.121226:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879728.121234:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1788879728.121234:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290192384/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788879728.121243:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879728.121250:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788879728.121252:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879728.121253:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879728.121258:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879728.121262:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879728.121268:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1788879728.121269:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290192256:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879728.121270:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879728.121273:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879728.121287:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879728.121289:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788879728.121294:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10666 00000100:00080000:3.0:1788879728.121296:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1788879728.121298:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788879728.121299:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879728.121304:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879728.121305:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879728.121307:0:178134:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1788879728.121313:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1788879728.121313:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879728.121315:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879728.121320:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879728.121321:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879728.121322:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879728.121324:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879728.121325:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879728.121333:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879728.121335:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879728.121337:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879728.121338:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879728.121339:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879728.121341:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879728.121343:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879728.121344:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879728.121350:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879728.121351:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121353:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879728.121353:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879728.121362:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879728.121363:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 02000000:00000001:3.0:1788879728.121368:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121372:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879728.121374:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879728.121379:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290192512/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879728.121384:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879728.121386:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879728.121388:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879728.121390:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879728.121393:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879728.121394:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879728.121396:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879728.121398:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879728.121400:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10666 00000100:00080000:3.0:1788879728.121410:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879728.121413:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879728.121414:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879728.121415:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879728.121416:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879728.121417:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879728.121419:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879728.121421:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879728.121422:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879728.121425:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879728.121432:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788879728.121434:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879728.121435:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879728.121441:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879728.121443:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879728.121444:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879728.121445:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879728.121447:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879728.121449:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879728.121453:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879728.121454:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121456:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879728.121456:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879728.121462:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879728.121464:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121467:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879728.121471:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121472:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121477:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:3.0:1788879728.121482:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290192640/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879728.121484:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121494:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305339945 00000100:00000001:3.0:1788879728.121504:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879728.121508:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121513:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879728.121513:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121528:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305339945 00000100:00000001:2.0:1788879728.121536:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1788879728.121541:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879728.121544:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121548:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879728.121557:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879728/real 1788879728] req@ffff94e50449f100 x1875765290192256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879728.121568:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879728.121571:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879728.121575:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879728.121578:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879728.121580:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.121582:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879728.121584:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879728.121597:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290192256/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879728.121605:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.121609:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879728.121612:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.121613:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879728.121615:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.121617:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879728.121620:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879728.121623:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879728.121627:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879728.121630:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879728.121634:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290192256:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879728.121639:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.121641:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879728.121643:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879728.121646:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879728.121651:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879728.121654:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879728.121657:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879728.121662:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879728.121664:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879728.121667:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879728.121669:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.121671:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.121679:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879728.121681:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121684:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879728.121686:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879728.121687:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879728.121689:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879728.121690:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879728.121693:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879728.121695:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879728.121697:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879728.121701:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290192384:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879728.121705:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879728.121708:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879728.121710:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879728.121712:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879728.121715:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121721:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879728.121723:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:2.0:1788879728.121728:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879728.121742:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879728.121746:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879728.121748:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879728.121750:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121752:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879728.121754:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121759:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305339403 00000100:00000001:2.0:1788879728.121762:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305339945 00000100:00000001:2.0:1788879728.121765:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305339403 00000100:00000001:2.0:1788879728.121768:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305339945 00000100:00000001:2.0:1788879728.121770:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879728.121772:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121774:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879728.121778:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879728/real 1788879728] req@ffff94e50449ca80 x1875765290192384/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879728.121787:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879728.121788:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879728.121790:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879728.121792:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879728.121793:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.121795:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879728.121796:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879728.121800:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290192384/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879728.121807:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.121810:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879728.121812:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.121814:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879728.121815:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.121817:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879728.121843:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879728.121847:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879728.121851:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879728.121853:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879728.121857:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290192384:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879728.121861:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879728.121863:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879728.121864:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879728.121867:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879728.121868:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879728.121870:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879728.121874:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290192512:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879728.121877:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879728.121879:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879728.121880:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879728.121882:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879728.121885:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121891:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879728.121893:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:2.0:1788879728.121897:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879728.121939:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879728.121944:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879728.121947:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879728.121948:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121950:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879728.121953:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121957:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305339402 00000100:00000001:2.0:1788879728.121960:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305339945 00000100:00000001:2.0:1788879728.121962:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305339402 00000100:00000001:2.0:1788879728.121964:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305339945 00000100:00000001:2.0:1788879728.121966:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879728.121969:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.121970:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879728.121976:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879728/real 1788879728] req@ffff94e50449c000 x1875765290192512/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879728.121984:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879728.121985:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879728.121987:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879728.121989:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879728.121990:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.121992:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879728.121993:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879728.121997:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290192512/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879728.122009:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.122011:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879728.122013:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.122015:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879728.122016:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.122018:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879728.122031:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879728.122033:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879728.122036:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879728.122039:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879728.122042:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290192512:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879728.122046:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879728.122047:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879728.122048:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879728.122051:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879728.122053:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879728.122054:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879728.122057:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290192640:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879728.122060:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879728.122063:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879728.122064:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879728.122066:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879728.122070:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.122074:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879728.122076:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:2.0:1788879728.122079:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879728.122089:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879728.122094:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879728.122096:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879728.122097:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.122099:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879728.122101:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.122105:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305339402 00000100:00000001:2.0:1788879728.122107:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305339945 00000100:00000001:2.0:1788879728.122109:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305339402 00000100:00000001:2.0:1788879728.122111:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305339945 00000100:00000001:2.0:1788879728.122113:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879728.122114:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.122116:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879728.122119:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879728/real 1788879728] req@ffff94e50449e300 x1875765290192640/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879728.122126:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879728.122127:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879728.122129:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879728.122130:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879728.122131:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.122133:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879728.122134:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879728.122138:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290192640/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879728.122144:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.122146:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879728.122148:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.122150:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879728.122151:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.122153:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879728.122165:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879728.122168:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879728.122171:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879728.122173:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879728.122177:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290192640:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879728.122180:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.122182:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879728.122183:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879728.122185:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879728.122187:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879728.122189:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879728.122190:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879728.122192:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879728.122193:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879728.122195:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879728.122196:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.122198:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879728.122199:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879728.122200:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879728.122202:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879728.122203:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879728.122204:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879728.122206:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879728.122207:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879728.122208:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879728.122209:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.122211:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879728.122212:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879728.122213:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879728.122215:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879728.122216:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879728.122217:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879728.122219:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879728.122220:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879728.122222:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879728.122223:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.122225:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879728.122228:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879728.122229:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879728.122231:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879728.122233:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879728.185008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879728.185014:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879728.185016:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879728.185020:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879728.185023:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879728.185027:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879728.185029:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879728.185031:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879728.185033:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879728.889014:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879728.889026:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879728.889031:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879728.889032:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879728.889049:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879728.889051:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879728.889053:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879729.208967:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879729.208972:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879729.208974:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879729.208978:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879729.208980:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879729.208984:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879729.208986:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879729.208988:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879729.208990:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879729.913031:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879729.913036:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879729.913041:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879729.913042:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879729.913235:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879729.913260:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879729.913266:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879730.232981:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879730.232987:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879730.232989:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879730.232993:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879730.232996:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879730.233000:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879730.233002:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879730.233004:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879730.233007:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879730.937025:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879730.937032:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879730.937037:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879730.937039:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879730.937152:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879730.937161:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879730.937169:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879731.257011:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879731.257027:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879731.257030:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879731.257035:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879731.257038:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879731.257042:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879731.257045:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879731.257047:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879731.257050:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879731.961065:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879731.961076:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879731.961083:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879731.961085:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879731.961112:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879731.961116:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879731.961120:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879732.281123:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879732.281132:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879732.281135:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879732.281142:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879732.281147:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879732.281152:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879732.281155:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879732.281159:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879732.281162:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879732.985024:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879732.985030:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879732.985036:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879732.985037:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879732.985056:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879732.985059:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879732.985062:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879733.241226:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879733.241238:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879733.241243:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879733.241249:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879733.241256:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879733.241268:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10672 00000100:00080000:3.0:1788879733.241311:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879733.241317:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879733.241321:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879733.241332:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879733.241336:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879733.241341:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879733.241346:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879733.241350:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879733.241354:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879733.241361:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879733.241366:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879733.241370:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879733.241374:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879733.241377:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.241380:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879733.241383:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879733.241393:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879733.241395:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.241399:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879733.241402:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879733.241407:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.241420:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879733.241437:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290192768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879733.241453:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879733.241461:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.241476:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879733.241482:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879733.241487:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879733.241490:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879733.241492:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879733.241495:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879733.241496:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879733.241501:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788879733.241503:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10672 00000100:00000001:1.0:1788879733.241527:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1788879733.241532:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879733.241533:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879733.241541:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.241542:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879733.241559:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879733.241559:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879733.241569:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788879733.241571:0:178134:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1788879733.241577:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879733.241578:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879733.241585:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00100000:1.0:1788879733.241593:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290192768:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879733.241603:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879733.241610:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879733.241614:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879733.241618:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879733.241624:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.241645:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879733.241648:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:1.0:1788879733.241653:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1788879733.241700:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000800:00000100:1.0:1788879733.241706:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879733.241708:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879733.241714:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879733.241723:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879733.241725:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879733.241729:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.241735:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788879733.241736:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879733.241742:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879733.241748:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879733.241749:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879733.241758:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.241760:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.241762:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879733.241771:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305345065 00000008:00100000:3.0:1788879733.241773:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879733.241783:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.241789:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.241835:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788879733.241837:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879733.241843:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.241846:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879733.241849:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879733.241852:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.241854:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305345065 00000100:00000001:3.0:1788879733.241862:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.241863:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879733.241870:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879733.241877:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879733.241879:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290192896/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879733.241885:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879733.241898:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879733.241903:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.241907:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879733.241910:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879733.241913:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879733.241915:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1788879733.241972:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879733/real 1788879733] req@ffff94e50449c700 x1875765290192768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879733.241978:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879733.241988:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879733.241999:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10672 00000400:00000001:1.0:1788879733.242011:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1788879733.242012:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000400:00000001:1.0:1788879733.242018:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879733.242020:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879733.242025:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879733.242028:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000020:00000001:3.0:1788879733.242032:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879733.242037:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879733.242041:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.242045:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879733.242047:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879733.242070:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290192768/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879733.242082:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.242087:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879733.242092:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.242095:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879733.242097:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.242124:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879733.242126:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879733.242129:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879733.242133:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879733.242135:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879733.242139:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290192768:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879733.242144:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.242146:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879733.242148:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879733.242151:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879733.242154:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879733.242156:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879733.242158:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879733.242162:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879733.242163:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879733.242166:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879733.242168:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.242171:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.242178:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879733.242180:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.242182:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879733.242184:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879733.242186:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879733.242187:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879733.242189:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879733.242192:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879733.242194:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879733.242196:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879733.242199:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290192896:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879733.242202:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879733.242205:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879733.242207:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879733.242209:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879733.242213:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.242220:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879733.242222:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000020:00000001:3.0:1788879733.242226:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879733.242230:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1788879733.242231:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879733.242237:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879733.242240:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879733.242242:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879733.242248:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:1.0:1788879733.242254:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879733.242255:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.242263:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879733.242266:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879733.242267:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.242270:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879733.242274:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.242279:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305344524 00000100:00000001:1.0:1788879733.242282:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305345065 00000100:00000001:1.0:1788879733.242285:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305344524 00000100:00000001:1.0:1788879733.242288:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305345065 00000100:00000001:1.0:1788879733.242290:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879733.242292:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.242294:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879733.242299:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879733/real 1788879733] req@ffff94e50449f480 x1875765290192896/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879733.242307:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879733.242309:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879733.242315:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879733.242318:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879733.242323:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879733.242325:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879733.242326:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879733.242331:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879733.242333:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788879733.242337:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290192896/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879733.242340:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.242347:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879733.242370:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1788879733.242371:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879733.242376:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:3.0:1788879733.242378:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.242389:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.242390:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.242396:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879733.242399:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879733.242402:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879733.242403:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 02000000:00000001:3.0:1788879733.242410:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.242415:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879733.242419:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879733.242428:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.242446:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.242447:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879733.242456:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879733.242481:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1788879733.242483:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290193024/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879733.242484:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879733.242492:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290192896:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879733.242505:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.242512:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879733.242515:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879733.242519:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788879733.242519:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879733.242526:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879733.242527:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879733.242531:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879733.242534:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788879733.242536:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879733.242538:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:3.0:1788879733.242560:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879733.242561:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879733.242566:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00080000:3.0:1788879733.242572:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879733.242573:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879733.242580:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879733.242582:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788879733.242586:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879733.242589:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879733.242593:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879733.242595:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879733.242603:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10672 00000100:00000001:1.0:1788879733.242604:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879733.242613:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788879733.242616:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879733.242617:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879733.242623:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879733.242626:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879733.242627:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.242631:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879733.242635:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1788879733.242641:0:178134:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1788879733.242642:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788879733.242649:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1788879733.242651:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879733.242656:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:1.0:1788879733.242661:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290193024:192.168.203.129@tcp:8:kworker.0 00000020:00000001:3.0:1788879733.242664:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879733.242673:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879733.242675:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879733.242680:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788879733.242681:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879733.242688:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879733.242695:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788879733.242696:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879733.242705:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.242706:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879733.242716:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879733.242722:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788879733.242723:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879733.242728:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 02000000:00000001:3.0:1788879733.242731:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.242739:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879733.242742:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:3.0:1788879733.242744:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879733.242749:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.242753:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879733.242762:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879733.242765:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.242774:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000100:1.0:1788879733.242775:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879733.242781:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879733.242787:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788879733.242789:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.242819:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879733.242824:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.242828:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879733.242831:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.242834:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.242844:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305344522 00000100:00100000:3.0:1788879733.242868:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290193152/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879733.242869:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305345065 00000100:00000001:1.0:1788879733.242876:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305344522 00000100:00000001:1.0:1788879733.242885:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305345065 00000100:00000001:1.0:1788879733.242895:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879733.242897:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879733.242901:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879733.242904:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.242908:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879733.242911:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879733.242977:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879733/real 1788879733] req@ffff94e50449f800 x1875765290193024/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879733.242994:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879733.242998:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879733.243007:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879733.243013:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879733.243015:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.243018:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879733.243021:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879733.243028:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290193024/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879733.243039:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.243045:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879733.243052:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.243055:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879733.243057:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.243060:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879733.243151:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879733.243154:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879733.243160:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879733.243164:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879733.243171:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290193024:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879733.243176:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.243178:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879733.243179:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879733.243181:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879733.243202:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879733.243205:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879733.243208:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879733.243215:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879733.243218:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879733.243223:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879733.243225:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.243227:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.243233:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879733.243235:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.243238:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879733.243240:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879733.243243:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879733.243247:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879733.243250:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879733.243255:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879733.243259:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879733.243263:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879733.243272:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290193152:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879733.243278:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879733.243281:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879733.243284:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879733.243304:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879733.243308:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.243321:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879733.243323:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:1.0:1788879733.243328:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879733.243342:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879733.243346:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879733.243349:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879733.243350:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.243352:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879733.243373:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.243377:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305344523 00000100:00000001:1.0:1788879733.243380:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305345065 00000100:00000001:1.0:1788879733.243384:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305344523 00000100:00000001:1.0:1788879733.243387:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305345065 00000100:00000001:1.0:1788879733.243389:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879733.243392:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.243394:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879733.243399:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879733/real 1788879733] req@ffff94e50449c380 x1875765290193152/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879733.243408:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879733.243410:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879733.243412:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879733.243414:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879733.243416:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.243418:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879733.243420:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879733.243425:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290193152/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879733.243433:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.243436:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879733.243439:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.243458:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879733.243459:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.243461:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879733.243476:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879733.243479:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879733.243482:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879733.243484:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879733.243487:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290193152:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879733.243490:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.243491:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879733.243493:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879733.243494:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879733.243495:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879733.243497:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879733.243498:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879733.243500:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879733.243501:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879733.243502:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879733.243503:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.243505:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.243525:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879733.243526:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.243529:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879733.243531:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879733.305041:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879733.305048:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879733.305050:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879733.305056:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879733.305060:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879733.305064:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.305068:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879733.305071:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879733.305074:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879734.009044:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879734.009053:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879734.009060:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879734.009062:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879734.009192:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879734.009201:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879734.009207:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879734.329136:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879734.329145:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879734.329150:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879734.329158:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879734.329162:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879734.329168:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879734.329171:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879734.329175:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879734.329179:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879735.032992:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879735.032999:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879735.033006:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879735.033009:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879735.033012:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879735.033016:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879735.033019:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879735.033022:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879735.033024:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879735.033027:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879735.033128:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879735.033136:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879735.033142:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879735.353016:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879735.353022:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879735.353025:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879735.353029:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879735.353031:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879735.353035:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879735.353038:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879735.353040:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879735.353042:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879736.056970:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879736.056975:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879736.056979:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879736.056980:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879736.056994:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879736.056996:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879736.056999:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879736.376969:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879736.376975:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879736.376976:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879736.376981:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879736.376984:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879736.376987:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879736.376989:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879736.376992:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879736.376994:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879737.081012:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879737.081037:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879737.081041:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879737.081046:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879737.081052:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879737.081056:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879737.081060:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879737.081064:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879737.081070:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879737.081073:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879737.081104:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879737.081108:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879737.081113:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879737.401011:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879737.401020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879737.401023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879737.401030:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879737.401035:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879737.401040:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879737.401043:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879737.401047:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879737.401050:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879738.105032:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879738.105039:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879738.105045:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879738.105046:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879738.105138:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879738.105144:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879738.105148:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879738.361041:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879738.361053:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879738.361056:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879738.361060:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879738.361065:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879738.361073:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10677 00000100:00080000:3.0:1788879738.361086:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879738.361089:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879738.361092:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879738.361101:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879738.361103:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879738.361107:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879738.361110:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879738.361112:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879738.361114:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879738.361119:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879738.361123:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879738.361126:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879738.361130:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879738.361134:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879738.361136:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879738.361138:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879738.361153:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879738.361156:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879738.361160:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879738.361165:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879738.361171:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879738.361187:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879738.361202:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290193280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879738.361219:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879738.361226:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879738.361249:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879738.361254:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879738.361258:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879738.361260:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879738.361262:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879738.361265:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879738.361269:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10677 00000100:00080000:3.0:1788879738.361274:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879738.361277:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879738.361279:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879738.361284:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879738.361286:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879738.361289:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879738.361292:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879738.361295:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879738.361298:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879738.361300:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1788879738.361303:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879738.361307:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879738.361308:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.361313:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788879738.361313:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879738.361319:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788879738.361323:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879738.361324:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879738.361333:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879738.361337:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879738.361342:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879738.361345:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1788879738.361346:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879738.361351:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879738.361352:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879738.361361:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879738.361364:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879738.361369:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879738.361372:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290193280:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879738.361373:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879738.361377:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879738.361382:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879738.361384:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879738.361388:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879738.361392:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879738.361394:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879738.361395:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879738.361405:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879738.361407:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290193408/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879738.361419:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879738.361421:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879738.361425:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879738.361428:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879738.361429:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879738.361434:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00080000:3.0:1788879738.361435:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879738.361441:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879738.361443:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879738.361445:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879738.361450:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879738.361453:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10677 00000100:00080000:3.0:1788879738.361456:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879738.361458:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879738.361460:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879738.361463:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879738.361466:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879738.361468:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879738.361471:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879738.361474:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879738.361475:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879738.361479:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879738.361481:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879738.361483:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879738.361487:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:1.0:1788879738.361488:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879738.361491:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879738.361496:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879738.361498:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879738.361501:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879738.361503:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.361505:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788879738.361510:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879738.361512:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788879738.361514:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.361518:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879738.361519:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879738.361523:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879738.361525:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879738.361527:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.361533:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879738.361535:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.361539:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305350185 00000100:00100000:3.0:1788879738.361545:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290193536/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879738.361554:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879738.361555:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879738.361562:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879738.361566:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.361568:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305350185 00000100:00080000:3.0:1788879738.361570:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879738.361576:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788879738.361577:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879738.361584:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879738.361585:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879738.361590:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879738.361591:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.361598:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879738.361606:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879738/real 1788879738] req@ffff94e50449d880 x1875765290193280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879738.361611:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879738.361619:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10677 00000400:00000001:1.0:1788879738.361628:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1788879738.361629:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000400:00000001:1.0:1788879738.361634:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879738.361636:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879738.361641:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879738.361641:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879738.361648:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000020:00000001:3.0:1788879738.361649:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879738.361651:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788879738.361653:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879738.361657:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1788879738.361658:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879738.361662:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1788879738.361664:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879738.361669:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879738.361673:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:1.0:1788879738.361674:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290193280/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879738.361682:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879738.361688:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879738.361689:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879738.361692:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879738.361694:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879738.361696:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879738.361700:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879738.361707:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.361711:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879738.361711:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879738.361722:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879738.361723:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1788879738.361725:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879738.361733:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879738.361736:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879738.361736:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879738.361740:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.361745:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879738.361747:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.361751:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.361773:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00100000:3.0:1788879738.361774:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290193664/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879738.361779:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879738.361792:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1788879738.361798:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879738.361803:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879738.361805:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879738.361808:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.361817:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879738.361825:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290193280:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879738.361832:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.361835:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879738.361837:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879738.361841:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879738.361846:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879738.361848:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879738.361851:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879738.361855:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879738.361857:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879738.361860:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879738.361862:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.361864:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.361872:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879738.361875:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.361877:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879738.361879:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879738.361880:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879738.361882:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879738.361884:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879738.361886:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879738.361888:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879738.361892:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879738.361895:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290193408:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879738.361898:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879738.361901:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879738.361940:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879738.361942:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879738.361946:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.361954:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879738.361955:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:1.0:1788879738.361960:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879738.361976:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879738.361982:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879738.361984:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879738.361986:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.361988:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879738.361991:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.361995:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305349644 00000100:00000001:1.0:1788879738.361999:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305350185 00000100:00000001:1.0:1788879738.362001:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305349644 00000100:00000001:1.0:1788879738.362004:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305350185 00000100:00000001:1.0:1788879738.362006:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879738.362008:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.362010:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879738.362015:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879738/real 1788879738] req@ffff94e50449fb80 x1875765290193408/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879738.362024:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879738.362025:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879738.362028:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879738.362030:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879738.362031:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362033:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879738.362035:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879738.362039:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290193408/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879738.362048:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362050:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879738.362053:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362054:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879738.362056:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362057:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879738.362074:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879738.362078:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879738.362082:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879738.362086:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879738.362091:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290193408:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879738.362097:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879738.362099:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879738.362102:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879738.362105:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879738.362109:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879738.362112:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879738.362117:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290193536:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879738.362123:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879738.362126:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879738.362129:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879738.362132:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879738.362137:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.362147:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879738.362150:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:1.0:1788879738.362156:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879738.362172:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879738.362177:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879738.362180:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879738.362182:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.362184:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879738.362187:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.362191:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305349642 00000100:00000001:1.0:1788879738.362194:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305350185 00000100:00000001:1.0:1788879738.362197:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305349642 00000100:00000001:1.0:1788879738.362199:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305350185 00000100:00000001:1.0:1788879738.362201:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879738.362203:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.362205:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879738.362209:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879738/real 1788879738] req@ffff94e50449d180 x1875765290193536/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879738.362217:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879738.362218:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879738.362220:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879738.362222:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879738.362224:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362225:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879738.362227:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879738.362231:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290193536/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879738.362238:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362241:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879738.362244:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362245:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879738.362247:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362248:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879738.362264:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879738.362267:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879738.362269:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879738.362271:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879738.362275:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290193536:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879738.362280:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879738.362281:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879738.362283:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879738.362285:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879738.362287:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879738.362289:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879738.362292:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290193664:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879738.362295:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879738.362297:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879738.362299:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879738.362301:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879738.362304:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.362308:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879738.362309:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:1.0:1788879738.362313:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879738.362322:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879738.362326:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879738.362328:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879738.362330:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.362332:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879738.362334:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.362337:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305349642 00000100:00000001:1.0:1788879738.362340:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305350185 00000100:00000001:1.0:1788879738.362342:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305349642 00000100:00000001:1.0:1788879738.362345:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305350185 00000100:00000001:1.0:1788879738.362347:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879738.362349:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.362350:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879738.362353:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879738/real 1788879738] req@ffff94e50449ce00 x1875765290193664/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879738.362368:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879738.362369:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879738.362371:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879738.362373:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879738.362374:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362376:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879738.362377:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879738.362381:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290193664/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879738.362387:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362389:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879738.362391:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362392:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879738.362394:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362395:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879738.362410:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879738.362413:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879738.362416:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879738.362418:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879738.362421:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290193664:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879738.362425:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362426:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879738.362428:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879738.362429:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879738.362431:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879738.362433:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879738.362451:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879738.362453:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879738.362455:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879738.362457:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879738.362459:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362461:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879738.362463:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879738.362464:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879738.362467:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879738.362468:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879738.362470:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879738.362471:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879738.362473:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879738.362474:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879738.362475:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362477:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879738.362478:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879738.362479:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879738.362481:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879738.362482:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879738.362483:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879738.362484:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879738.362485:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879738.362487:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879738.362488:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362490:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.362494:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879738.362495:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.362497:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879738.362516:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879738.424995:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879738.425003:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879738.425006:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879738.425011:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879738.425016:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879738.425020:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.425023:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879738.425026:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879738.425030:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879739.129035:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879739.129044:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879739.129051:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879739.129053:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879739.129156:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879739.129164:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879739.129171:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879739.449063:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879739.449075:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879739.449079:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879739.449087:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879739.449093:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879739.449098:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879739.449102:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879739.449108:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879739.449112:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879740.153073:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879740.153086:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879740.153098:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879740.153101:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879740.153139:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879740.153144:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879740.153150:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879740.473072:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879740.473081:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879740.473085:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879740.473093:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879740.473097:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879740.473103:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879740.473107:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879740.473110:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879740.473114:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879741.177014:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879741.177021:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879741.177029:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879741.177031:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879741.177058:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879741.177061:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879741.177091:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879741.496987:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879741.496994:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879741.496996:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879741.497001:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879741.497005:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879741.497009:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879741.497012:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879741.497015:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879741.497017:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879742.201114:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879742.201123:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879742.201130:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879742.201132:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879742.201284:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879742.201295:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879742.201303:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879742.520996:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879742.521003:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879742.521005:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879742.521009:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879742.521012:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879742.521015:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879742.521018:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879742.521020:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879742.521022:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879743.225042:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879743.225050:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879743.225057:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879743.225059:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879743.225155:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879743.225161:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879743.225165:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879743.481038:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879743.481056:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879743.481061:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879743.481067:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879743.481074:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879743.481084:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10682 00000100:00080000:3.0:1788879743.481097:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879743.481100:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879743.481104:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879743.481113:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879743.481117:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879743.481121:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879743.481124:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879743.481127:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879743.481129:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879743.481134:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879743.481138:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879743.481141:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879743.481146:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879743.481151:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879743.481154:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879743.481157:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879743.481169:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879743.481172:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879743.481176:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879743.481180:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879743.481186:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879743.481199:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879743.481211:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290193792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879743.481225:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879743.481231:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879743.481255:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879743.481259:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879743.481264:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879743.481266:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879743.481269:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879743.481273:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879743.481276:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10682 00000100:00080000:3.0:1788879743.481281:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879743.481284:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879743.481286:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879743.481290:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879743.481292:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879743.481296:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879743.481299:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879743.481302:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879743.481304:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879743.481310:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879743.481313:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879743.481316:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879743.481324:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788879743.481325:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879743.481333:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.481335:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879743.481337:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879743.481343:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1788879743.481346:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879743.481349:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:3.0:1788879743.481352:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.481356:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879743.481358:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879743.481368:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879743.481369:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879743.481377:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879743.481377:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879743.481380:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879743.481386:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879743.481388:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879743.481392:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879743.481402:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290193792:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879743.481403:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.481419:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1788879743.481420:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290193920/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879743.481433:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879743.481436:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879743.481443:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788879743.481444:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879743.481451:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879743.481453:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879743.481459:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788879743.481463:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879743.481468:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879743.481472:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1788879743.481473:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879743.481484:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879743.481489:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10682 00000100:00080000:3.0:1788879743.481494:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879743.481496:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879743.481498:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879743.481502:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879743.481503:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1788879743.481505:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879743.481510:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000020:00000001:3.0:1788879743.481512:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879743.481522:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879743.481523:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879743.481527:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879743.481532:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879743.481536:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879743.481539:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879743.481542:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879743.481546:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879743.481549:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879743.481551:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879743.481553:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879743.481556:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879743.481559:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879743.481578:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879743.481582:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879743.481587:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879743.481591:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879743.481593:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879743.481596:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788879743.481597:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.481607:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879743.481609:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.481613:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.481619:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:3.0:1788879743.481623:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290194048/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879743.481628:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.481639:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305355305 00000100:00000001:3.0:1788879743.481657:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879743.481659:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879743.481661:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879743.481670:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879743.481673:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879743.481677:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879743.481679:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879743.481681:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879743.481685:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879743.481686:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305355305 00000100:00000001:1.0:1788879743.481695:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788879743.481701:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10682 00000100:00000001:1.0:1788879743.481705:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879743.481711:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879743.481712:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.481744:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879743.481747:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879743.481756:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:1.0:1788879743.481758:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879743/real 1788879743] req@ffff94e50449e680 x1875765290193792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788879743.481766:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879743.481772:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879743.481779:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879743.481786:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000400:00000001:1.0:1788879743.481787:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879743.481795:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879743.481796:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879743.481802:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879743.481805:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:3.0:1788879743.481811:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879743.481814:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879743.481818:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.481821:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879743.481824:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879743.481827:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1788879743.481832:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879743.481833:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879743.481840:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879743.481842:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879743.481845:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879743.481854:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879743.481859:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879743.481860:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290193792/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879743.481866:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879743.481872:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879743.481881:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879743.481884:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879743.481885:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879743.481893:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.481896:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879743.481937:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.481940:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879743.481943:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.481946:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879743.481949:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879743.481952:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879743.481955:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879743.481968:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00100000:3.0:1788879743.481977:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290194176/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879743.481978:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879743.482000:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290193792:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879743.482014:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879743.482018:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879743.482023:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.482029:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879743.482035:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.482037:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879743.482047:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879743.482055:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879743.482060:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879743.482063:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879743.482070:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879743.482075:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879743.482080:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879743.482083:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482087:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482097:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879743.482099:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.482102:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879743.482105:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879743.482107:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879743.482109:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879743.482111:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879743.482114:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879743.482117:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879743.482120:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879743.482125:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290193920:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879743.482130:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879743.482133:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879743.482137:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879743.482139:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879743.482143:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.482153:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879743.482155:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:1.0:1788879743.482161:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879743.482181:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879743.482188:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879743.482191:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879743.482193:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.482195:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879743.482199:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.482204:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305354764 00000100:00000001:1.0:1788879743.482208:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305355305 00000100:00000001:1.0:1788879743.482212:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305354764 00000100:00000001:1.0:1788879743.482215:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305355305 00000100:00000001:1.0:1788879743.482218:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879743.482220:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.482222:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879743.482228:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879743/real 1788879743] req@ffff94e50449df80 x1875765290193920/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879743.482238:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879743.482240:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879743.482243:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879743.482245:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879743.482246:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482249:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879743.482251:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879743.482256:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290193920/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879743.482264:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482267:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879743.482271:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482273:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879743.482274:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482277:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879743.482295:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879743.482299:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879743.482303:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879743.482306:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879743.482311:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290193920:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879743.482315:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879743.482317:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879743.482319:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879743.482321:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879743.482323:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879743.482326:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879743.482330:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290194048:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879743.482334:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879743.482336:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879743.482338:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879743.482340:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879743.482343:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.482349:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879743.482350:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:1.0:1788879743.482354:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879743.482366:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879743.482370:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879743.482372:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879743.482375:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.482377:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879743.482380:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.482384:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305354762 00000100:00000001:1.0:1788879743.482387:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305355305 00000100:00000001:1.0:1788879743.482390:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305354762 00000100:00000001:1.0:1788879743.482393:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305355305 00000100:00000001:1.0:1788879743.482396:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879743.482398:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.482400:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879743.482405:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879743/real 1788879743] req@ffff94e50449ce00 x1875765290194048/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879743.482414:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879743.482415:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879743.482417:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879743.482419:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879743.482421:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482423:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879743.482424:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879743.482428:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290194048/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879743.482437:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482439:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879743.482442:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482444:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879743.482446:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482448:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879743.482464:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879743.482467:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879743.482471:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879743.482473:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879743.482477:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290194048:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879743.482482:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879743.482483:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879743.482486:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879743.482489:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879743.482491:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879743.482493:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879743.482497:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290194176:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879743.482500:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879743.482502:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879743.482504:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879743.482505:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879743.482509:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.482513:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879743.482515:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:1.0:1788879743.482518:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879743.482528:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879743.482532:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879743.482534:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879743.482536:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.482538:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879743.482542:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.482545:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305354762 00000100:00000001:1.0:1788879743.482548:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305355305 00000100:00000001:1.0:1788879743.482551:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305354762 00000100:00000001:1.0:1788879743.482554:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305355305 00000100:00000001:1.0:1788879743.482557:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879743.482559:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.482561:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879743.482565:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879743/real 1788879743] req@ffff94e50449d180 x1875765290194176/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879743.482574:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879743.482576:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879743.482578:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879743.482580:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879743.482581:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482584:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879743.482586:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879743.482590:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290194176/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879743.482599:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482602:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879743.482604:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482606:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879743.482608:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482610:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879743.482625:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879743.482628:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879743.482631:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879743.482634:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879743.482637:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290194176:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879743.482641:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482644:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879743.482645:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879743.482648:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879743.482650:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879743.482652:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879743.482654:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879743.482657:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879743.482659:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879743.482662:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879743.482664:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482666:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879743.482667:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879743.482669:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879743.482672:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879743.482673:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879743.482675:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879743.482677:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879743.482678:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879743.482680:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879743.482682:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482684:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879743.482686:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879743.482687:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879743.482689:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879743.482691:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879743.482692:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879743.482694:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879743.482695:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879743.482697:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879743.482699:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482701:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.482704:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879743.482706:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.482708:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879743.482710:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879743.545077:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879743.545085:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879743.545088:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879743.545094:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879743.545099:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879743.545104:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.545108:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879743.545111:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879743.545115:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879744.249091:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879744.249101:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879744.249111:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879744.249114:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879744.249149:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879744.249153:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879744.249157:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879744.569051:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879744.569061:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879744.569064:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879744.569072:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879744.569077:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879744.569083:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879744.569087:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879744.569091:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879744.569094:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879745.273001:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879745.273007:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879745.273013:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879745.273016:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879745.273018:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879745.273021:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879745.273024:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879745.273036:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879745.273038:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879745.273040:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879745.273058:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879745.273060:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879745.273063:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879745.592993:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879745.593001:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879745.593004:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879745.593010:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879745.593015:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879745.593020:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879745.593024:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879745.593027:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879745.593031:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879746.297076:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879746.297086:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879746.297096:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879746.297099:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879746.297242:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879746.297251:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879746.297258:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879746.617059:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879746.617067:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879746.617070:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879746.617076:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879746.617080:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879746.617085:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879746.617088:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879746.617091:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879746.617095:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879747.321037:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879747.321047:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879747.321050:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879747.321054:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879747.321061:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879747.321064:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879747.321067:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879747.321070:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879747.321074:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879747.321076:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879747.321161:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879747.321168:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879747.321173:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879747.641003:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879747.641011:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879747.641014:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879747.641020:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879747.641025:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879747.641030:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879747.641033:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879747.641036:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879747.641040:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879748.345036:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879748.345041:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879748.345047:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879748.345048:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879748.345065:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879748.345068:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879748.345070:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879748.601000:0:178259:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879748.601009:0:178259:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879748.601012:0:178259:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879748.601016:0:178259:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879748.601020:0:178259:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879748.601026:0:178259:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10687 00000100:00080000:2.0:1788879748.601053:0:178259:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879748.601056:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879748.601059:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879748.601067:0:178259:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879748.601070:0:178259:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879748.601074:0:178259:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879748.601076:0:178259:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879748.601078:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879748.601079:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879748.601083:0:178259:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879748.601085:0:178259:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879748.601088:0:178259:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879748.601092:0:178259:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879748.601095:0:178259:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601097:0:178259:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879748.601100:0:178259:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879748.601111:0:178259:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879748.601112:0:178259:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601115:0:178259:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879748.601118:0:178259:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879748.601139:0:178259:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601151:0:178259:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879748.601162:0:178259:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5500 x1875765290194304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879748.601173:0:178259:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879748.601178:0:178259:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601192:0:178259:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879748.601196:0:178259:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879748.601200:0:178259:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879748.601202:0:178259:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879748.601203:0:178259:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879748.601206:0:178259:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879748.601208:0:178259:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10687 00000100:00080000:2.0:1788879748.601212:0:178259:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879748.601215:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879748.601216:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879748.601219:0:178259:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879748.601220:0:178259:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879748.601222:0:178259:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879748.601224:0:178259:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879748.601226:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879748.601227:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879748.601230:0:178259:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879748.601232:0:178259:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879748.601233:0:178259:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879748.601239:0:178259:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879748.601241:0:178259:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601243:0:178259:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879748.601246:0:178259:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879748.601249:0:178259:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601252:0:178259:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879748.601256:0:178259:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879748.601258:0:178259:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601260:0:178259:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879748.601261:0:178259:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879748.601263:0:178259:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601268:0:178259:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601273:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1788879748.601274:0:178259:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5c00 x1875765290194432/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879748.601280:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879748.601285:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879748.601286:0:178259:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879748.601290:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879748.601291:0:178259:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601295:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601297:0:178259:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879748.601302:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1788879748.601302:0:178259:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879748.601310:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1788879748.601310:0:178259:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879748.601315:0:178259:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1788879748.601317:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879748.601322:0:178259:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788879748.601329:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290194304:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879748.601330:0:178259:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879748.601339:0:178259:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10687 00000100:00000001:1.0:1788879748.601342:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1788879748.601347:0:178259:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788879748.601350:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788879748.601353:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879748.601356:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601358:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879748.601364:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1788879748.601366:0:178259:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879748.601371:0:178259:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1788879748.601374:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879748.601374:0:178259:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879748.601383:0:178259:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879748.601385:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879748.601387:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879748.601391:0:178259:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879748.601394:0:178259:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879748.601396:0:178259:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879748.601399:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1788879748.601400:0:178259:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879748.601406:0:178259:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601407:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074553600 : -117763634998016 : ffff94e5045e5500) 00000100:00000001:2.0:1788879748.601409:0:178259:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879748.601413:0:178259:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879748.601417:0:178259:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601418:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879748.601421:0:178259:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879748.601430:0:178259:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879748.601433:0:178259:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601436:0:178259:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879748.601438:0:178259:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879748.601442:0:178259:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601449:0:178259:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879748.601458:0:178259:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7100 x1875765290194560/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879748.601470:0:178259:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879748.601492:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601492:0:178259:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601496:0:178259:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601500:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305360425 00000100:00080000:2.0:1788879748.601502:0:178259:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879748.601509:0:178259:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879748.601511:0:178259:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879748.601513:0:178259:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879748.601517:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788879748.601518:0:178259:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879748.601523:0:178259:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10687 00000100:00080000:2.0:1788879748.601529:0:178259:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879748.601531:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305360425 00000100:00000001:2.0:1788879748.601532:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879748.601535:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879748.601540:0:178259:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879748.601541:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1788879748.601542:0:178259:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879748.601546:0:178259:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879748.601550:0:178259:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879748.601551:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879748.601555:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879748.601575:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601576:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879748.601584:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879748.601586:0:178259:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879748.601592:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879748.601594:0:178259:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879748.601600:0:178259:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879748.601604:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1788879748.601606:0:178259:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879748.601610:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:2.0:1788879748.601613:0:178259:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601616:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879748.601619:0:178259:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879748.601622:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1788879748.601625:0:178259:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879748.601627:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879748.601631:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:2.0:1788879748.601632:0:178259:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601636:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601637:0:178259:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879748.601646:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879748.601663:0:178259:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879748.601665:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879748.601666:0:178259:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601670:0:178259:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879748.601674:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879748.601674:0:178259:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879748.601678:0:178259:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601698:0:178259:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879748.601699:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290194432:192.168.203.129@tcp:38:kworker.0 00000100:00100000:2.0:1788879748.601705:0:178259:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4a80 x1875765290194688/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879748.601709:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879748.601714:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879748.601718:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879748.601723:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879748.601728:0:178259:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879748.601730:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601731:0:178259:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879748.601738:0:178259:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601743:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879748.601747:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074555392 : -117763634996224 : ffff94e5045e5c00) 00000100:00000001:1.0:1788879748.601753:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879748.601769:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601774:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305359883 00000100:00000001:1.0:1788879748.601777:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305360425 00000100:00000001:1.0:1788879748.601779:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305359883 00000100:00000001:1.0:1788879748.601781:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305360425 00000100:00000001:1.0:1788879748.601783:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879748.601785:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601787:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879748.601788:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879748.601789:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879748.601791:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879748.601792:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879748.601794:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879748.601797:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290194560:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879748.601800:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879748.601802:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879748.601803:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879748.601805:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879748.601807:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601812:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879748.601813:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074560768 : -117763634990848 : ffff94e5045e7100) 00000100:00000001:1.0:1788879748.601816:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879748.601823:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601826:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305359881 00000100:00000001:1.0:1788879748.601829:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305360425 00000100:00000001:1.0:1788879748.601830:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305359881 00000100:00000001:1.0:1788879748.601832:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305360425 00000100:00000001:1.0:1788879748.601834:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879748.601835:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601837:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601839:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879748.601841:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879748.601842:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788879748.601844:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879748.601845:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879748.601846:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879748.601848:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879748.601849:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879748.601851:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879748.601853:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879748.601854:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879748.601857:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290194688:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879748.601859:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879748.601860:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879748.601861:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879748.601862:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879748.601865:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601867:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879748.601868:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550912 : -117763635000704 : ffff94e5045e4a80) 00000100:00000001:1.0:1788879748.601871:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879748.601877:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601879:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305359881 00000100:00000001:1.0:1788879748.601881:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305360425 00000100:00000001:1.0:1788879748.601883:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305359881 00000100:00000001:1.0:1788879748.601885:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305360425 00000100:00000001:1.0:1788879748.601886:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879748.601887:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601889:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601890:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879748.601891:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879748.601892:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788879748.601893:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879748.601894:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879748.601895:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.601897:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879748.665052:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879748.665059:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879748.665061:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879748.665066:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879748.665070:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879748.665074:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879748.665077:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879748.665080:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879748.665083:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879749.369121:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879749.369131:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879749.369142:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879749.369146:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879749.369179:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879749.369184:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879749.369189:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879749.688993:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879749.688999:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879749.689002:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879749.689006:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879749.689010:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879749.689014:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879749.689033:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879749.689035:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879749.689038:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879750.393018:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879750.393029:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879750.393040:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879750.393042:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879750.393145:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879750.393155:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879750.393162:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879750.713025:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879750.713034:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879750.713037:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879750.713042:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879750.713047:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879750.713052:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879750.713056:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879750.713061:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879750.713066:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879751.417109:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879751.417122:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879751.417132:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879751.417136:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879751.417296:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879751.417302:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879751.417307:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788879751.673377:0:2352:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1788879751.673446:0:2352:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788879751.673473:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879751.673494:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879751.673524:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879751.673529:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879751.673534:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879751.673554:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879751.673564:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879751.673569:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879751.673572:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879751.673575:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879751.673583:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879751.673594:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879751.673604:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879751.673608:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879751.673611:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879751.673623:0:2352:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:1.0:1788879751.673632:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879751.673643:0:2352:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879751.673644:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879751.673657:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879751.673662:0:2352:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879751.673693:0:2352:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879751.673704:0:2352:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:1.0:1788879751.673706:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879748/real 1788879751] req@ffff94e5045e5c00 x1875765290194432/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879751.673732:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879751.673737:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879751.673750:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879751.673755:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879751.673758:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.673762:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879751.673764:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879751.673772:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5c00 x1875765290194432/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879751.673788:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.673794:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879751.673802:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.673805:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879751.673807:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.673812:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879751.673839:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879751.673846:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879751.673855:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879751.673860:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879751.673867:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290194432:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879751.673875:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879751.673884:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879748/real 1788879751] req@ffff94e5045e7100 x1875765290194560/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879751.673943:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879751.673948:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879751.673955:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879751.673958:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879751.673961:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.673966:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879751.673969:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879751.673978:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7100 x1875765290194560/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879751.673993:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.673996:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879751.674001:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.674005:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879751.674007:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.674010:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879751.674035:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879751.674041:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879751.674049:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879751.674055:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879751.674061:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290194560:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879751.674070:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879751.674079:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879748/real 1788879751] req@ffff94e5045e5500 x1875765290194304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879751.674097:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879751.674100:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879751.674107:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879751.674110:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879751.674113:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.674116:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879751.674119:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879751.674127:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5500 x1875765290194304/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879751.674140:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.674145:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879751.674149:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.674152:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879751.674154:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.674157:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879751.674160:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879751.674165:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879751.674172:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879751.674176:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879751.674182:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290194304:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879751.674190:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879751.674194:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879751.674198:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879751.674203:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879751.674213:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879751.674218:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879751.674222:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879751.674228:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879751.674230:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879751.674236:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879751.674238:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.674242:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879751.674244:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879751.674246:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879751.674251:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879751.674253:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879751.674255:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879751.674258:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879751.674260:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879751.674263:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879751.674265:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.674269:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879751.674272:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879751.674274:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879751.674277:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879751.674280:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879751.674282:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879751.674286:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879751.674291:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879751.674296:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879751.674298:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.674303:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879751.674308:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879751.674312:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879751.674315:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879751.674326:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879748/real 1788879751] req@ffff94e5045e4a80 x1875765290194688/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879751.674342:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879751.674345:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879751.674352:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879751.674357:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879751.674360:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.674364:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879751.674367:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879751.674376:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4a80 x1875765290194688/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879751.674389:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.674393:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879751.674398:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.674400:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879751.674403:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.674406:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879751.674429:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879751.674434:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879751.674441:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879751.674446:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879751.674453:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290194688:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879751.674461:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.674465:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879751.674467:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879751.674471:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879751.674477:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879751.674481:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879751.674484:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879751.674490:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879751.674493:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879751.674496:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879751.674499:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.674503:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.674514:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879751.674518:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879751.674522:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879751.674525:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879751.737105:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879751.737113:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879751.737116:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879751.737123:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879751.737127:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879751.737132:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.737136:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879751.737140:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879751.737143:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879752.441054:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879752.441066:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879752.441073:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879752.441075:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879752.441105:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879752.441109:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879752.441112:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879752.761011:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879752.761020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879752.761023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879752.761031:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879752.761036:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879752.761041:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879752.761044:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879752.761046:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879752.761049:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879753.465001:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879753.465009:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879753.465014:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879753.465016:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879753.465035:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879753.465037:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879753.465040:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879753.721064:0:178259:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879753.721081:0:178259:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879753.721085:0:178259:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879753.721091:0:178259:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879753.721099:0:178259:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879753.721110:0:178259:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10692 00000100:00080000:2.0:1788879753.721128:0:178259:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879753.721132:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879753.721135:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879753.721148:0:178259:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879753.721152:0:178259:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879753.721157:0:178259:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879753.721160:0:178259:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879753.721163:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879753.721165:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879753.721172:0:178259:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879753.721175:0:178259:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879753.721179:0:178259:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879753.721184:0:178259:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879753.721188:0:178259:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879753.721191:0:178259:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879753.721195:0:178259:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879753.721213:0:178259:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879753.721215:0:178259:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879753.721220:0:178259:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879753.721224:0:178259:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879753.721231:0:178259:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879753.721250:0:178259:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879753.721263:0:178259:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5180 x1875765290194816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879753.721279:0:178259:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879753.721285:0:178259:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879753.721310:0:178259:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879753.721314:0:178259:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879753.721319:0:178259:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879753.721321:0:178259:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879753.721323:0:178259:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879753.721327:0:178259:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879753.721330:0:178259:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10692 00000100:00080000:2.0:1788879753.721335:0:178259:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879753.721337:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879753.721339:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879753.721343:0:178259:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879753.721345:0:178259:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879753.721349:0:178259:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879753.721352:0:178259:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879753.721355:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879753.721358:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879753.721365:0:178259:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879753.721370:0:178259:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879753.721373:0:178259:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879753.721384:0:178259:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879753.721387:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879753.721391:0:178259:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879753.721399:0:178259:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879753.721400:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879753.721406:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:2.0:1788879753.721407:0:178259:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879753.721411:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:2.0:1788879753.721416:0:178259:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.721419:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879753.721427:0:178259:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879753.721431:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879753.721443:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879753.721446:0:178259:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879753.721449:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879753.721454:0:178259:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879753.721467:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290194816:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879753.721469:0:178259:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879753.721481:0:178259:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879753.721484:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879753.721491:0:178259:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879753.721493:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879753.721500:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879753.721505:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879753.721511:0:178259:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879753.721514:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879753.721533:0:178259:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e5880 x1875765290194944/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879753.721542:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879753.721552:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074552704 : -117763634998912 : ffff94e5045e5180) 00000100:00000001:1.0:1788879753.721563:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879753.721583:0:178259:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879753.721589:0:178259:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879753.721601:0:178259:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879753.721609:0:178259:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788879753.721620:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788879753.721624:0:178259:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879753.721631:0:178259:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879753.721635:0:178259:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879753.721638:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879753.721671:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879753.721676:0:178259:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879753.721677:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.721689:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:2.0:1788879753.721697:0:178259:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10692 00000100:00000001:1.0:1788879753.721705:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879753.721712:0:178259:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879753.721720:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305365545 00000100:00000001:2.0:1788879753.721723:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879753.721730:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879753.721744:0:178259:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879753.721748:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1788879753.721759:0:178259:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879753.721773:0:178259:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879753.721779:0:178259:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879753.721783:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879753.721786:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879753.721795:0:178259:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879753.721798:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305365545 00000100:00000001:2.0:1788879753.721805:0:178259:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879753.721815:0:178259:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879753.721816:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1788879753.721827:0:178259:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879753.721829:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879753.721843:0:178259:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.721845:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879753.721856:0:178259:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879753.721860:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000008:00100000:2.0:1788879753.721870:0:178259:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879753.721886:0:178259:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879753.721889:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879753/real 1788879753] req@ffff94e5045e5180 x1875765290194816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879753.721898:0:178259:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879753.721995:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1788879753.722003:0:178259:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000400:00000001:1.0:1788879753.722004:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1788879753.722012:0:178259:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.722021:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879753.722023:0:178259:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879753.722031:0:178259:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879753.722034:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:2.0:1788879753.722040:0:178259:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.722043:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.722054:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879753.722060:0:178259:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.722061:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879753.722084:0:178259:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290195072/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879753.722087:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5180 x1875765290194816/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879753.722162:0:178259:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879753.722166:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879753.722176:0:178259:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.722181:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879753.722189:0:178259:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.722195:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1788879753.722200:0:178259:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879753.722208:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1788879753.722218:0:178259:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879753.722219:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879753.722228:0:178259:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879753.722233:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1788879753.722238:0:178259:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879753.722243:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879753.722251:0:178259:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879753.722259:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879753.722269:0:178259:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10692 00000100:00080000:1.0:1788879753.722282:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00080000:2.0:1788879753.722287:0:178259:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879753.722294:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1788879753.722297:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879753.722305:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:1.0:1788879753.722315:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290194816:192.168.203.129@tcp:250:kworker.0 00000020:00000001:2.0:1788879753.722321:0:178259:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879753.722329:0:178259:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879753.722337:0:178259:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879753.722339:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1788879753.722355:0:178259:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879753.722356:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879753.722362:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788879753.722370:0:178259:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879753.722372:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1788879753.722379:0:178259:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879753.722386:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879753.722395:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00080000:2.0:1788879753.722398:0:178259:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000020:00000001:1.0:1788879753.722403:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:2.0:1788879753.722409:0:178259:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879753.722422:0:178259:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879753.722423:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879753.722432:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1788879753.722436:0:178259:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879753.722441:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 02000000:00000001:2.0:1788879753.722447:0:178259:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.722452:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879753.722457:0:178259:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879753.722465:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1788879753.722467:0:178259:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879753.722479:0:178259:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879753.722483:0:178259:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.722485:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879753.722495:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879753.722496:0:178259:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879753.722503:0:178259:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.722510:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879753.722513:0:178259:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879753.722520:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1788879753.722522:0:178259:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879753.722529:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1788879753.722532:0:178259:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.722541:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879753.722549:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879753.722552:0:178259:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879753.722557:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879753.722567:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1788879753.722577:0:178259:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290195200/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879753.722583:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879753.722606:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290194944:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879753.722613:0:178259:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879753.722626:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1788879753.722627:0:178259:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879753.722636:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1788879753.722643:0:178259:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879753.722647:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879753.722676:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879753.722689:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.722706:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879753.722710:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074554496 : -117763634997120 : ffff94e5045e5880) 00000100:00000001:1.0:1788879753.722720:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879753.722759:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879753.722770:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879753.722776:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879753.722779:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.722783:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879753.722789:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.722799:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305365004 00000100:00000001:1.0:1788879753.722805:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305365545 00000100:00000001:1.0:1788879753.722809:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305365004 00000100:00000001:1.0:1788879753.722814:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305365545 00000100:00000001:1.0:1788879753.722819:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879753.722823:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.722827:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879753.722837:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879753/real 1788879753] req@ffff94e5045e5880 x1875765290194944/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879753.722855:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879753.722858:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879753.722864:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879753.722868:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879753.722871:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.722875:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879753.722878:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879753.722886:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e5880 x1875765290194944/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879753.722901:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.722950:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879753.722957:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.722961:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879753.722964:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.722969:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879753.723000:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879753.723006:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879753.723016:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879753.723021:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879753.723028:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e5880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290194944:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879753.723038:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879753.723042:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879753.723046:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879753.723051:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879753.723056:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879753.723061:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879753.723069:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290195072:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879753.723075:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879753.723079:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879753.723083:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879753.723087:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879753.723093:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.723105:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879753.723108:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:1.0:1788879753.723127:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879753.723160:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879753.723171:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879753.723175:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879753.723178:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.723182:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879753.723189:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.723197:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305365003 00000100:00000001:1.0:1788879753.723203:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305365545 00000100:00000001:1.0:1788879753.723209:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305365003 00000100:00000001:1.0:1788879753.723215:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305365545 00000100:00000001:1.0:1788879753.723219:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879753.723223:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.723227:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879753.723237:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879753/real 1788879753] req@ffff94e5045e6a00 x1875765290195072/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879753.723256:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879753.723259:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879753.723265:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879753.723269:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879753.723271:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.723275:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879753.723278:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879753.723287:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290195072/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879753.723324:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.723329:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879753.723333:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.723336:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879753.723339:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.723343:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879753.723368:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879753.723374:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879753.723381:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879753.723386:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879753.723392:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290195072:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879753.723400:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.723404:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879753.723406:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879753.723410:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879753.723415:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879753.723418:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879753.723422:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879753.723427:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879753.723429:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879753.723434:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879753.723436:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.723440:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879753.723442:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879753.723445:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879753.723449:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879753.723451:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879753.723453:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879753.723456:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879753.723458:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879753.723461:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879753.723464:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.723467:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.723476:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879753.723480:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.723484:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879753.723487:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879753.723490:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879753.723493:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879753.723496:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879753.723500:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879753.723503:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879753.723507:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879753.723514:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290195200:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879753.723521:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879753.723525:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879753.723528:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879753.723532:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879753.723537:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.723547:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879753.723551:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:1.0:1788879753.723558:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879753.723600:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879753.723610:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879753.723615:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879753.723618:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.723621:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879753.723627:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.723634:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305365003 00000100:00000001:1.0:1788879753.723639:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305365545 00000100:00000001:1.0:1788879753.723644:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305365003 00000100:00000001:1.0:1788879753.723675:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305365545 00000100:00000001:1.0:1788879753.723681:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879753.723685:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.723690:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879753.723699:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879753/real 1788879753] req@ffff94e5045e6680 x1875765290195200/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879753.723727:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879753.723729:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879753.723745:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879753.723747:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879753.723749:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.723752:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879753.723754:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879753.723760:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290195200/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879753.723769:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.723772:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879753.723775:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.723777:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879753.723779:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.723782:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879753.723800:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879753.723803:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879753.723807:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879753.723809:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879753.723813:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290195200:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879753.723817:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.723820:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879753.723822:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879753.723824:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879753.723828:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879753.723830:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879753.723832:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879753.723835:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879753.723837:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879753.723839:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879753.723841:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.723844:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.723848:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879753.723850:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.723853:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879753.723855:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879753.785083:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879753.785092:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879753.785095:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879753.785102:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879753.785106:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879753.785112:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.785115:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879753.785119:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879753.785122:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879754.489017:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879754.489026:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879754.489033:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879754.489035:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879754.489140:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879754.489152:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879754.489161:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879754.808997:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879754.809006:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879754.809009:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879754.809016:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879754.809020:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879754.809025:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879754.809029:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879754.809043:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879754.809048:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879755.513000:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879755.513006:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879755.513010:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879755.513012:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879755.513014:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879755.513017:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879755.513019:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879755.513020:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879755.513022:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879755.513023:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879755.513096:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879755.513101:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879755.513105:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879755.833013:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879755.833018:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879755.833020:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879755.833024:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879755.833027:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879755.833030:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879755.833032:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879755.833034:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879755.833036:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879756.536990:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879756.536997:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879756.537002:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879756.537003:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879756.537020:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879756.537022:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879756.537025:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879756.856985:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879756.856992:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879756.856995:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879756.857000:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879756.857004:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879756.857009:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879756.857013:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879756.857016:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879756.857019:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879757.560982:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879757.560989:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879757.560991:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879757.560994:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879757.560998:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879757.561000:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879757.561002:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879757.561003:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879757.561006:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879757.561008:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879757.561025:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879757.561028:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879757.561031:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879757.880992:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879757.881001:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879757.881004:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879757.881010:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879757.881014:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879757.881018:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879757.881020:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879757.881023:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879757.881027:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879758.585024:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879758.585031:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879758.585039:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879758.585040:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879758.585145:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879758.585151:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879758.585156:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879758.841092:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879758.841104:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879758.841107:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879758.841111:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879758.841116:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879758.841124:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10697 00000100:00080000:3.0:1788879758.841136:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879758.841138:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879758.841140:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879758.841149:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879758.841152:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879758.841155:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879758.841157:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879758.841159:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879758.841160:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879758.841164:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879758.841166:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879758.841169:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879758.841173:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879758.841176:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879758.841179:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879758.841182:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879758.841197:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879758.841200:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879758.841204:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879758.841208:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879758.841214:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879758.841231:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879758.841243:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290195328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879758.841252:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879758.841257:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879758.841274:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879758.841278:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879758.841281:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879758.841283:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879758.841285:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879758.841289:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879758.841291:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10697 00000100:00080000:3.0:1788879758.841296:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879758.841298:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879758.841300:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879758.841304:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879758.841306:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879758.841309:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879758.841311:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879758.841314:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879758.841316:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879758.841321:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879758.841324:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879758.841326:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879758.841330:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788879758.841334:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879758.841337:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.841338:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879758.841339:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879758.841344:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1788879758.841346:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879758.841349:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:3.0:1788879758.841349:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879758.841353:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.841354:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879758.841357:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879758.841359:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879758.841361:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879758.841362:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879758.841363:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879758.841365:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879758.841369:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879758.841371:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879758.841377:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879758.841379:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290195456/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879758.841387:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290195328:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879758.841388:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879758.841412:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879758.841421:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.841423:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1788879758.841427:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879758.841434:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:1.0:1788879758.841435:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788879758.841436:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879758.841443:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1788879758.841445:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879758.841448:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879758.841457:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10697 00000100:00080000:3.0:1788879758.841462:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879758.841465:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879758.841466:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879758.841470:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879758.841472:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1788879758.841475:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1788879758.841476:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879758.841483:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879758.841486:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879758.841488:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879758.841490:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879758.841493:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879758.841502:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879758.841505:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879758.841508:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879758.841512:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879758.841514:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879758.841517:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879758.841520:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879758.841523:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.841528:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788879758.841530:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879758.841536:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.841538:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:3.0:1788879758.841540:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879758.841544:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879758.841551:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.841555:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879758.841559:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879758.841567:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290195584/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879758.841577:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879758.841579:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879758.841582:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879758.841585:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879758.841590:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879758.841592:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879758.841594:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879758.841598:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879758.841602:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10697 00000100:00080000:3.0:1788879758.841607:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879758.841609:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879758.841611:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879758.841644:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879758.841647:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879758.841651:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879758.841655:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879758.841659:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879758.841663:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879758.841672:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000800:00000100:1.0:1788879758.841681:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879758.841683:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879758.841692:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879758.841699:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879758.841704:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879758.841709:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879758.841711:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000008:00100000:3.0:1788879758.841714:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879758.841723:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.841724:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879758.841728:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.841733:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879758.841740:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879758.841742:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879758.841745:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879758.841751:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879758.841754:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879758.841759:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879758.841760:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.841777:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305370665 00000100:00000001:3.0:1788879758.841779:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879758.841797:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290195712/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879758.841803:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879758.841823:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879758.841829:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879758.841833:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.841849:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305370665 00000100:00000001:1.0:1788879758.841864:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879758.841873:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879758.841879:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.841886:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879758.841901:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879758/real 1788879758] req@ffff94e50449fb80 x1875765290195328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879758.841991:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879758.841996:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879758.842005:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879758.842011:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879758.842015:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.842021:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879758.842026:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879758.842051:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290195328/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879758.842060:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.842064:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879758.842068:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.842070:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879758.842072:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.842075:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879758.842078:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879758.842082:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879758.842087:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879758.842090:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879758.842094:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290195328:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879758.842100:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.842103:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879758.842104:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879758.842107:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879758.842113:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879758.842115:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879758.842118:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879758.842123:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879758.842125:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879758.842129:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879758.842131:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.842134:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.842143:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879758.842145:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.842148:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879758.842149:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879758.842151:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879758.842175:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879758.842178:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879758.842181:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879758.842183:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879758.842186:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879758.842190:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290195456:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879758.842194:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879758.842197:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879758.842198:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879758.842201:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879758.842204:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.842212:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879758.842214:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:1.0:1788879758.842221:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879758.842240:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879758.842249:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879758.842253:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879758.842255:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.842257:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879758.842260:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.842265:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305370124 00000100:00000001:1.0:1788879758.842269:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305370665 00000100:00000001:1.0:1788879758.842271:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305370124 00000100:00000001:1.0:1788879758.842274:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305370665 00000100:00000001:1.0:1788879758.842277:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879758.842279:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.842281:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879758.842285:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879758/real 1788879758] req@ffff94e50449d880 x1875765290195456/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879758.842295:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879758.842298:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879758.842302:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879758.842304:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879758.842325:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.842329:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879758.842332:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879758.842341:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290195456/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879758.842355:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.842360:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879758.842365:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.842368:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879758.842371:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.842374:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879758.842398:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879758.842404:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879758.842411:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879758.842416:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879758.842423:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290195456:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879758.842431:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879758.842434:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879758.842437:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879758.842441:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879758.842446:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879758.842451:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879758.842457:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290195584:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879758.842464:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879758.842468:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879758.842471:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879758.842475:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879758.842482:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.842492:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879758.842495:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:1.0:1788879758.842502:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879758.842527:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879758.842535:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879758.842539:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879758.842542:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.842546:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879758.842551:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.842557:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305370122 00000100:00000001:1.0:1788879758.842562:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305370665 00000100:00000001:1.0:1788879758.842567:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305370122 00000100:00000001:1.0:1788879758.842571:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305370665 00000100:00000001:1.0:1788879758.842575:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879758.842579:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.842583:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879758.842591:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879758/real 1788879758] req@ffff94e50449c380 x1875765290195584/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879758.842607:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879758.842610:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879758.842644:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879758.842648:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879758.842651:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.842655:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879758.842658:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879758.842667:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290195584/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879758.842693:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.842699:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879758.842703:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.842706:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879758.842709:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.842713:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879758.842734:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879758.842739:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879758.842746:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879758.842751:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879758.842758:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290195584:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879758.842766:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879758.842769:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879758.842771:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879758.842776:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879758.842779:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879758.842784:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879758.842790:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290195712:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879758.842797:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879758.842800:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879758.842803:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879758.842807:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879758.842813:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.842823:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879758.842826:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:1.0:1788879758.842832:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879758.842858:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879758.842868:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879758.842872:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879758.842875:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.842879:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879758.842885:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.842892:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305370122 00000100:00000001:1.0:1788879758.842898:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305370665 00000100:00000001:1.0:1788879758.842946:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305370122 00000100:00000001:1.0:1788879758.842954:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305370665 00000100:00000001:1.0:1788879758.842958:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879758.842963:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.842967:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879758.842976:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879758/real 1788879758] req@ffff94e50449f800 x1875765290195712/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879758.842993:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879758.842995:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879758.843001:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879758.843005:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879758.843008:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.843011:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879758.843014:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879758.843022:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290195712/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879758.843053:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.843055:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879758.843057:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.843059:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879758.843060:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.843062:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879758.843076:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879758.843079:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879758.843082:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879758.843085:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879758.843089:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290195712:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879758.843093:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.843094:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879758.843096:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879758.843098:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879758.843100:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879758.843102:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879758.843103:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879758.843106:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879758.843107:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879758.843109:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879758.843111:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.843113:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879758.843114:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879758.843115:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879758.843118:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879758.843119:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879758.843121:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879758.843123:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879758.843124:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879758.843126:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879758.843127:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.843129:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879758.843130:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879758.843132:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879758.843133:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879758.843134:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879758.843135:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879758.843138:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879758.843139:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879758.843141:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879758.843142:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.843144:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.843148:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879758.843150:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.843152:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879758.843153:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879758.905064:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879758.905071:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879758.905073:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879758.905080:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879758.905084:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879758.905088:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.905090:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879758.905093:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879758.905095:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879759.609128:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879759.609138:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879759.609145:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879759.609147:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879759.609259:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879759.609266:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879759.609271:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879759.929062:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879759.929073:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879759.929076:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879759.929084:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879759.929090:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879759.929096:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879759.929100:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879759.929104:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879759.929108:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879760.633111:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879760.633121:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879760.633128:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879760.633130:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879760.633162:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879760.633165:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879760.633170:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879760.953046:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879760.953056:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879760.953059:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879760.953066:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879760.953072:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879760.953077:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879760.953080:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879760.953084:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879760.953088:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879761.657082:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879761.657091:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879761.657098:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879761.657100:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879761.657130:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879761.657134:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879761.657137:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879761.977068:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879761.977079:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879761.977082:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879761.977090:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879761.977095:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879761.977101:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879761.977105:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879761.977109:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879761.977113:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879762.681123:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879762.681132:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879762.681139:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879762.681141:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879762.681247:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879762.681256:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879762.681263:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879763.001012:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879763.001021:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879763.001024:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879763.001031:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879763.001036:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879763.001042:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879763.001046:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879763.001050:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879763.001053:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879763.705095:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879763.705103:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879763.705108:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879763.705110:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879763.705202:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879763.705211:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879763.705218:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879763.961124:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879763.961136:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879763.961140:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879763.961145:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879763.961151:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879763.961160:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10702 00000100:00080000:1.0:1788879763.961174:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879763.961178:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879763.961181:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879763.961191:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879763.961195:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879763.961199:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879763.961202:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879763.961205:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879763.961207:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879763.961213:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879763.961216:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879763.961220:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879763.961225:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879763.961229:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879763.961232:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879763.961235:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879763.961249:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879763.961251:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879763.961255:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879763.961259:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879763.961266:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879763.961280:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879763.961293:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100bb80 x1875765290195840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879763.961308:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879763.961314:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879763.961340:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879763.961345:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879763.961349:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879763.961351:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879763.961353:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879763.961357:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879763.961360:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10702 00000100:00080000:1.0:1788879763.961364:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879763.961367:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879763.961370:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879763.961375:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879763.961377:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879763.961379:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879763.961383:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879763.961386:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879763.961389:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879763.961390:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879763.961398:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1788879763.961400:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879763.961405:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879763.961409:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.961411:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788879763.961415:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879763.961418:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879763.961428:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879763.961429:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879763.961433:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879763.961435:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879763.961443:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879763.961444:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879763.961454:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879763.961458:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290195840:192.168.203.129@tcp:250:kworker.0 00000008:00000001:1.0:1788879763.961462:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.961474:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788879763.961475:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879763.961484:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879763.961486:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879763.961490:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879763.961492:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879763.961497:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1788879763.961499:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879763.961504:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879763.961507:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879763.961509:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879763.961517:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879763.961528:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100aa00 x1875765290195968/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879763.961538:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879763.961540:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879763.961548:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.961548:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286523264 : -117763423028352 : ffff94e51100bb80) 00000100:00000001:1.0:1788879763.961556:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879763.961562:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1788879763.961564:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1788879763.961567:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879763.961572:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879763.961574:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879763.961608:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879763.961612:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10702 00000100:00080000:1.0:1788879763.961617:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879763.961619:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879763.961622:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879763.961627:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879763.961629:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879763.961632:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879763.961636:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879763.961639:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879763.961642:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879763.961647:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879763.961650:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879763.961651:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879763.961656:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879763.961662:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788879763.961665:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879763.961667:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879763.961674:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879763.961679:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879763.961683:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.961690:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:1.0:1788879763.961691:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879763.961699:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.961703:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879763.961704:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879763.961711:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879763.961712:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879763.961715:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.961718:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879763.961726:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.961730:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879763.961739:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b480 x1875765290196096/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879763.961741:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305375785 00000100:00000001:1.0:1788879763.961758:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879763.961763:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.961763:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788879763.961771:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879763.961776:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879763.961779:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879763.961781:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879763.961783:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879763.961787:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879763.961790:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305375785 00000100:00080000:1.0:1788879763.961792:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10702 00000100:00000001:2.0:1788879763.961805:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1788879763.961806:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879763.961815:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879763.961817:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879763.961818:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879763.961828:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879763.961830:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879763.961838:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879763.961838:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:1.0:1788879763.961848:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:1.0:1788879763.961852:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:2.0:1788879763.961857:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879763/real 1788879763] req@ffff94e51100bb80 x1875765290195840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879763.961858:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879763.961865:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879763.961879:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879763.961886:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879763.961896:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879763.961946:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879763.961951:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879763.961955:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000400:00000001:2.0:1788879763.961957:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879763.961963:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00100000:1.0:1788879763.961966:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879763.961974:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.961975:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879763.961982:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879763.961989:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879763.961992:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.961993:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879763.961997:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879763.961999:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788879763.962003:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879763.962006:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1788879763.962009:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962011:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879763.962020:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879763.962024:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100bb80 x1875765290195840/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1788879763.962034:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100b800 x1875765290196224/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879763.962058:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879763.962066:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879763.962068:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879763.962076:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962078:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879763.962084:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962085:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879763.962091:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962095:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879763.962099:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879763.962104:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879763.962111:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879763.962115:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879763.962123:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290195840:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879763.962131:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962134:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879763.962136:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879763.962140:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879763.962144:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879763.962147:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879763.962149:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879763.962154:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879763.962157:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879763.962160:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879763.962163:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962166:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962174:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879763.962177:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962180:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879763.962183:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879763.962185:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879763.962187:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879763.962189:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879763.962191:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879763.962194:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879763.962198:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879763.962202:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290195968:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879763.962206:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879763.962209:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879763.962220:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879763.962223:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879763.962227:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962236:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879763.962237:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286518784 : -117763423032832 : ffff94e51100aa00) 00000100:00000001:2.0:1788879763.962244:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879763.962263:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879763.962269:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879763.962272:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879763.962274:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962277:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879763.962280:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962285:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305375244 00000100:00000001:2.0:1788879763.962289:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305375785 00000100:00000001:2.0:1788879763.962293:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305375244 00000100:00000001:2.0:1788879763.962296:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305375785 00000100:00000001:2.0:1788879763.962299:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879763.962302:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962304:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879763.962310:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879763/real 1788879763] req@ffff94e51100aa00 x1875765290195968/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879763.962323:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879763.962325:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879763.962328:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879763.962330:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879763.962332:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962334:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879763.962336:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879763.962341:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100aa00 x1875765290195968/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879763.962350:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962353:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879763.962356:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962358:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879763.962359:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962362:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879763.962381:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879763.962384:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879763.962388:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879763.962390:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879763.962394:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290195968:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879763.962399:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879763.962400:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879763.962402:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879763.962405:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879763.962407:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879763.962410:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879763.962414:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290196096:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879763.962417:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879763.962419:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879763.962422:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879763.962424:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879763.962428:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962433:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879763.962435:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286521472 : -117763423030144 : ffff94e51100b480) 00000100:00000001:2.0:1788879763.962440:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879763.962452:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879763.962457:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879763.962460:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879763.962461:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962464:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879763.962467:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962470:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305375242 00000100:00000001:2.0:1788879763.962473:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305375785 00000100:00000001:2.0:1788879763.962477:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305375242 00000100:00000001:2.0:1788879763.962480:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305375785 00000100:00000001:2.0:1788879763.962482:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879763.962484:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962486:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879763.962491:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879763/real 1788879763] req@ffff94e51100b480 x1875765290196096/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879763.962501:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879763.962502:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879763.962505:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879763.962506:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879763.962508:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962510:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879763.962512:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879763.962519:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b480 x1875765290196096/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879763.962534:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962538:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879763.962544:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962548:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879763.962550:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962553:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879763.962597:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879763.962604:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879763.962612:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879763.962617:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879763.962624:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290196096:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879763.962632:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879763.962635:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879763.962638:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879763.962642:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879763.962645:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879763.962649:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879763.962654:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290196224:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879763.962659:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879763.962661:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879763.962664:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879763.962666:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879763.962671:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962680:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879763.962682:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286522368 : -117763423029248 : ffff94e51100b800) 00000100:00000001:2.0:1788879763.962687:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879763.962705:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879763.962711:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879763.962714:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879763.962716:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962718:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879763.962722:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962726:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305375242 00000100:00000001:2.0:1788879763.962730:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305375785 00000100:00000001:2.0:1788879763.962733:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305375242 00000100:00000001:2.0:1788879763.962737:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305375785 00000100:00000001:2.0:1788879763.962740:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879763.962742:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962745:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879763.962750:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879763/real 1788879763] req@ffff94e51100b800 x1875765290196224/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879763.962759:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879763.962761:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879763.962764:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879763.962766:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879763.962768:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962771:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879763.962773:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879763.962777:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100b800 x1875765290196224/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879763.962787:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962789:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879763.962792:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962794:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879763.962796:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962798:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879763.962815:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879763.962818:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879763.962821:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879763.962824:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879763.962828:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290196224:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879763.962833:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962835:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879763.962838:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879763.962840:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879763.962843:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879763.962845:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879763.962846:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879763.962850:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879763.962852:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879763.962855:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879763.962857:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962859:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879763.962860:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879763.962862:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879763.962865:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879763.962866:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879763.962868:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879763.962870:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879763.962871:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879763.962873:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879763.962874:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962876:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879763.962878:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879763.962879:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879763.962881:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879763.962882:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879763.962883:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879763.962885:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879763.962886:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879763.962888:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879763.962889:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962891:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879763.962896:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879763.962897:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879763.962900:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879763.962934:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879764.025020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879764.025025:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879764.025027:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879764.025032:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879764.025035:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879764.025038:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879764.025041:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879764.025044:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879764.025046:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879764.729061:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879764.729069:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879764.729088:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879764.729090:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879764.729114:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879764.729117:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879764.729121:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879765.049005:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879765.049014:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879765.049017:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879765.049024:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879765.049029:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879765.049035:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879765.049039:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879765.049042:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879765.049046:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879765.753123:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879765.753133:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879765.753140:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879765.753146:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879765.753150:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879765.753155:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879765.753159:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879765.753162:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879765.753165:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879765.753168:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879765.753233:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879765.753239:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879765.753245:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879766.073012:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879766.073021:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879766.073024:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879766.073030:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879766.073035:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879766.073040:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879766.073043:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879766.073046:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879766.073049:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879766.777088:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879766.777097:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879766.777104:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879766.777106:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879766.777223:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879766.777231:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879766.777239:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879767.097071:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879767.097080:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879767.097083:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879767.097090:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879767.097095:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879767.097100:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879767.097104:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879767.097107:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879767.097110:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879767.801070:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879767.801087:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879767.801092:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879767.801099:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879767.801109:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879767.801113:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879767.801118:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879767.801122:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879767.801128:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879767.801131:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879767.801227:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879767.801233:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879767.801240:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879768.121043:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879768.121055:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879768.121060:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879768.121070:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879768.121077:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879768.121084:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879768.121089:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879768.121093:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879768.121098:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879768.825086:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879768.825110:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879768.825120:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879768.825123:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879768.825204:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879768.825211:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879768.825218:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879769.081177:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879769.081196:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879769.081200:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879769.081206:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879769.081214:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879769.081225:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10707 00000100:00080000:3.0:1788879769.081243:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879769.081248:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879769.081251:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879769.081264:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879769.081267:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879769.081273:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879769.081277:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879769.081280:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879769.081283:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879769.081291:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879769.081295:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879769.081299:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879769.081304:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879769.081308:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879769.081312:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879769.081315:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879769.081329:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879769.081331:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879769.081336:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879769.081340:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879769.081358:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879769.081377:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879769.081391:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290196352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879769.081408:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879769.081415:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879769.081454:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879769.081461:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879769.081466:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879769.081469:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879769.081471:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879769.081476:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879769.081481:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10707 00000100:00080000:3.0:1788879769.081502:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879769.081506:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879769.081509:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879769.081517:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879769.081521:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879769.081525:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879769.081531:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788879769.081531:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879769.081564:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879769.081569:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879769.081575:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879769.081578:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879769.081602:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1788879769.081606:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879769.081610:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879769.081616:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879769.081622:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879769.081625:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879769.081630:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879769.081635:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879769.081639:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788879769.081646:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290196352:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879769.081652:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879769.081658:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879769.081661:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879769.081663:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879769.081670:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.081688:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879769.081690:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:1.0:1788879769.081694:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879769.081735:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879769.081750:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788879769.081755:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.081757:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879769.081763:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879769.081764:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.081769:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000008:00100000:3.0:1788879769.081771:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879769.081778:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.081780:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879769.081787:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.081789:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305380905 02000000:00000001:3.0:1788879769.081797:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879769.081801:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.081803:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879769.081806:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879769.081810:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879769.081814:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.081823:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305380905 00000100:00000001:3.0:1788879769.081825:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.081857:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879769.081863:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879769.081867:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.081871:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879769.081883:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879769/real 1788879769] req@ffff94e50449f480 x1875765290196352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879769.081899:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879769.081901:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879769.081921:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879769.081924:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879769.081927:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.081931:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879769.081933:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879769.081982:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290196352/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879769.081997:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.082002:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879769.082007:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.082011:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879769.082015:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.082018:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879769.082037:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879769.082041:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879769.082045:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879769.082047:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879769.082051:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290196352:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879769.082055:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.082057:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879769.082059:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879769.082062:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879769.082067:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879769.082069:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879769.082071:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879769.082075:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879769.082077:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879769.082081:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879769.082082:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.082084:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.082091:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879769.082094:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.082096:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879769.082098:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879769.082182:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290196480/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879769.082203:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879769.082207:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879769.082228:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879769.082234:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879769.082239:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879769.082243:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879769.082247:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879769.082252:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879769.082260:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10707 00000100:00080000:3.0:1788879769.082270:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879769.082273:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879769.082278:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879769.082283:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879769.082287:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879769.082290:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879769.082296:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879769.082301:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879769.082304:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879769.082312:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290196480:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879769.082318:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879769.082324:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879769.082328:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879769.082332:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879769.082336:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.082366:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879769.082371:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:1.0:1788879769.082380:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879769.082394:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879769.082397:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879769.082404:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879769.082411:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:1.0:1788879769.082418:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:3.0:1788879769.082421:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879769.082428:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879769.082429:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879769.082454:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879769.082460:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879769.082464:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1788879769.082474:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879769.082475:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.082496:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879769.082496:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879769.082510:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879769.082516:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879769.082517:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879769.082537:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.082562:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305380364 00000100:00000001:3.0:1788879769.082580:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879769.082589:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879769.082591:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305380905 00000008:00000001:3.0:1788879769.082601:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.082602:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305380364 00000100:00000001:3.0:1788879769.082611:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.082613:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305380905 00000100:00000001:1.0:1788879769.082625:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879769.082628:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.082631:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1788879769.082631:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879769.082639:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879769.082643:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879769/real 1788879769] req@ffff94e50449c700 x1875765290196480/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879769.082649:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879769.082660:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:1.0:1788879769.082664:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879769.082668:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788879769.082670:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.082674:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879769.082677:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879769.082679:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879769.082683:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.082684:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879769.082690:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1788879769.082712:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290196608/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879769.082714:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290196480/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879769.082803:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879769.082804:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879769.082826:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.082840:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879769.082844:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.082857:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788879769.082859:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879769.082867:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1788879769.082870:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879769.082871:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879769.082879:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879769.082881:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1788879769.082887:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879769.082895:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879769.082917:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879769.082927:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879769.082949:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10707 00000100:00080000:3.0:1788879769.082965:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879769.082971:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879769.082975:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879769.082994:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00080000:1.0:1788879769.082996:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000020:00000001:3.0:1788879769.083004:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879769.083006:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:3.0:1788879769.083013:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:1.0:1788879769.083015:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290196480:192.168.203.129@tcp:38:kworker.0 00000020:00000001:3.0:1788879769.083031:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879769.083032:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.083042:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879769.083049:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1788879769.083050:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879769.083052:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1788879769.083057:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879769.083077:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:3.0:1788879769.083080:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000020:00000001:1.0:1788879769.083082:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879769.083089:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788879769.083091:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879769.083095:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879769.083103:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:3.0:1788879769.083103:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879769.083115:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879769.083117:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.083123:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879769.083124:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879769.083137:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1788879769.083139:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.083141:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.083155:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879769.083157:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879769.083159:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879769.083167:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1788879769.083168:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879769.083172:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879769.083181:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1788879769.083183:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879769.083189:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879769.083193:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879769.083198:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879769.083200:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879769.083202:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879769.083213:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879769.083215:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879769.083219:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879769.083230:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879769.083239:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290196736/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879769.083245:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879769.083270:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290196608:192.168.203.129@tcp:8:kworker.0 00000100:00000001:3.0:1788879769.083273:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879769.083283:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879769.083284:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879769.083290:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879769.083296:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879769.083296:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879769.083312:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879769.083317:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.083327:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879769.083330:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:1.0:1788879769.083336:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879769.083379:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879769.083386:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879769.083390:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879769.083392:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.083394:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879769.083398:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.083403:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305380363 00000100:00000001:1.0:1788879769.083406:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305380905 00000100:00000001:1.0:1788879769.083409:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305380363 00000100:00000001:1.0:1788879769.083412:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305380905 00000100:00000001:1.0:1788879769.083415:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879769.083418:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.083420:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879769.083426:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879769/real 1788879769] req@ffff94e50449e300 x1875765290196608/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879769.083436:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879769.083438:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879769.083441:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879769.083443:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879769.083445:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.083447:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879769.083466:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879769.083471:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290196608/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879769.083479:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.083481:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879769.083485:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.083488:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879769.083490:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.083494:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879769.083517:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879769.083523:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879769.083530:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879769.083533:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879769.083556:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290196608:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879769.083565:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.083569:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879769.083572:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879769.083575:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879769.083580:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879769.083583:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879769.083585:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879769.083589:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879769.083592:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879769.083612:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879769.083614:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.083616:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.083621:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879769.083622:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.083624:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879769.083626:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879769.083627:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879769.083629:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879769.083631:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879769.083633:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879769.083635:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879769.083637:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879769.083640:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290196736:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879769.083643:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879769.083645:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879769.083648:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879769.083649:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879769.083652:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.083657:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879769.083659:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:1.0:1788879769.083662:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879769.083674:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879769.083678:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879769.083680:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879769.083682:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.083683:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879769.083685:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.083688:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305380363 00000100:00000001:1.0:1788879769.083691:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305380905 00000100:00000001:1.0:1788879769.083693:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305380363 00000100:00000001:1.0:1788879769.083695:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305380905 00000100:00000001:1.0:1788879769.083696:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879769.083698:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.083699:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879769.083703:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879769/real 1788879769] req@ffff94e50449c000 x1875765290196736/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879769.083709:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879769.083710:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879769.083712:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879769.083713:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879769.083714:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.083716:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879769.083718:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879769.083721:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290196736/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879769.083726:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.083728:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879769.083730:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.083732:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879769.083733:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.083735:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879769.083750:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879769.083752:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879769.083755:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879769.083757:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879769.083759:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290196736:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879769.083762:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.083764:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879769.083765:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879769.083766:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879769.083767:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879769.083768:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879769.083770:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879769.083771:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879769.083773:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879769.083774:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879769.083775:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.083777:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.083779:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879769.083780:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.083782:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879769.083783:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879769.145026:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879769.145033:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879769.145035:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879769.145041:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879769.145044:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879769.145048:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.145051:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879769.145054:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879769.145056:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879769.849058:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879769.849071:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879769.849083:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879769.849086:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879769.849156:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879769.849161:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879769.849166:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879770.169032:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879770.169042:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879770.169047:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879770.169055:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879770.169060:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879770.169067:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879770.169071:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879770.169075:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879770.169080:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879770.872947:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879770.872952:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879770.872956:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879770.872956:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879770.873064:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879770.873069:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879770.873073:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879771.193004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879771.193010:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879771.193012:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879771.193015:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879771.193018:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879771.193022:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879771.193024:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879771.193027:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879771.193028:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879771.897054:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879771.897065:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879771.897072:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879771.897075:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879771.897220:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879771.897228:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879771.897237:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879772.216985:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879772.216991:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879772.216993:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879772.216998:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879772.217001:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879772.217004:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879772.217007:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879772.217009:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879772.217011:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879772.921039:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879772.921048:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879772.921067:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879772.921069:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879772.921134:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879772.921138:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879772.921142:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879773.240986:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879773.240991:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879773.240993:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879773.240998:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879773.241000:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879773.241003:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879773.241006:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879773.241008:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879773.241010:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879773.944996:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879773.945001:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879773.945006:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879773.945007:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879773.945023:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879773.945025:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879773.945028:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879774.201018:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879774.201040:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879774.201044:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879774.201048:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879774.201054:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879774.201062:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10713 00000100:00080000:3.0:1788879774.201072:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879774.201076:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879774.201079:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879774.201087:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879774.201090:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879774.201104:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879774.201107:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879774.201110:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879774.201111:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879774.201116:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879774.201119:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879774.201122:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879774.201126:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879774.201131:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879774.201133:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879774.201136:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879774.201147:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879774.201149:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879774.201153:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879774.201156:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879774.201161:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879774.201174:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879774.201184:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290196864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879774.201197:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879774.201202:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879774.201217:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879774.201221:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879774.201225:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879774.201227:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879774.201229:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879774.201233:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879774.201236:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10713 00000100:00080000:3.0:1788879774.201239:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879774.201242:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879774.201244:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879774.201248:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879774.201250:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879774.201253:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879774.201254:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1788879774.201259:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879774.201260:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879774.201263:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879774.201264:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879774.201267:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879774.201268:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879774.201271:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879774.201275:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788879774.201277:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879774.201281:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879774.201282:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879774.201288:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879774.201288:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1788879774.201295:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290196864:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788879774.201296:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879774.201302:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879774.201303:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879774.201307:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1788879774.201308:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879774.201310:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879774.201314:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:3.0:1788879774.201314:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879774.201320:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879774.201322:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879774.201323:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879774.201331:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879774.201332:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879774.201334:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879774.201336:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879774.201338:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201344:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1788879774.201345:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201349:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00100000:3.0:1788879774.201354:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290196992/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879774.201356:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879774.201378:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879774.201380:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879774.201384:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879774.201386:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879774.201390:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879774.201391:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879774.201393:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879774.201396:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879774.201399:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10713 00000100:00080000:3.0:1788879774.201403:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879774.201405:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879774.201408:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:1.0:1788879774.201410:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000020:00000001:3.0:1788879774.201415:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879774.201418:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879774.201420:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879774.201423:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879774.201424:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879774.201428:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879774.201429:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879774.201432:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879774.201434:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201437:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:3.0:1788879774.201438:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879774.201442:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879774.201443:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879774.201448:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879774.201449:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305386025 02000000:00000001:3.0:1788879774.201451:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879774.201454:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879774.201467:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879774.201469:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879774.201470:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00000001:3.0:1788879774.201472:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879774.201476:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201478:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305386025 02000000:00000001:3.0:1788879774.201482:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879774.201484:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1788879774.201485:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201488:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879774.201490:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879774.201491:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879774.201493:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879774.201495:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1788879774.201497:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879774.201502:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879774/real 1788879774] req@ffff94e50449ca80 x1875765290196864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879774.201517:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879774.201528:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290197120/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1788879774.201529:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879774.201532:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879774.201536:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879774.201539:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879774.201542:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879774.201543:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879774.201545:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879774.201547:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201549:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879774.201553:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879774.201569:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290196864/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879774.201569:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879774.201580:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879774.201584:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879774.201585:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879774.201588:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1788879774.201589:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879774.201594:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879774.201595:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879774.201597:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879774.201599:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788879774.201601:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10713 00000100:00000001:1.0:1788879774.201602:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879774.201605:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879774.201609:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879774.201610:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879774.201614:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00080000:1.0:1788879774.201615:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879774.201619:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879774.201622:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290196864:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879774.201625:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879774.201626:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879774.201627:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879774.201630:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879774.201632:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879774.201635:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879774.201636:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879774.201638:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:3.0:1788879774.201640:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879774.201641:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000020:00000001:3.0:1788879774.201646:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788879774.201647:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1788879774.201651:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 02000000:00000001:1.0:1788879774.201652:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879774.201654:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879774.201657:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879774.201657:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879774.201661:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879774.201662:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879774.201668:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879774.201669:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879774.201673:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879774.201673:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879774.201678:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879774.201679:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879774.201682:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879774.201685:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788879774.201685:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879774.201687:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879774.201691:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879774.201691:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879774.201693:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879774.201695:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879774.201696:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879774.201700:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788879774.201700:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879774.201706:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290196992:192.168.203.129@tcp:38:kworker.0 00000008:00000001:3.0:1788879774.201707:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201713:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879774.201713:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879774.201716:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879774.201718:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879774.201719:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879774.201720:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879774.201722:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879774.201724:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879774.201727:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879774.201729:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788879774.201729:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879774.201732:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 02000000:00000001:3.0:1788879774.201734:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201739:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879774.201742:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879774.201751:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290197248/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788879774.201754:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879774.201759:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879774.201762:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879774.201764:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201768:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879774.201769:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879774.201771:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879774.201773:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201776:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305385483 00000100:00000001:3.0:1788879774.201778:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201780:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305386025 00000100:00000001:1.0:1788879774.201784:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305385483 00000100:00000001:1.0:1788879774.201786:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305386025 00000100:00000001:1.0:1788879774.201787:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879774.201789:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201790:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879774.201793:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879774/real 1788879774] req@ffff94e50449f100 x1875765290196992/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879774.201799:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879774.201800:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879774.201802:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879774.201804:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879774.201805:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.201807:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879774.201808:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879774.201810:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290196992/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879774.201816:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.201818:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879774.201819:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.201820:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879774.201821:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.201822:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879774.201835:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879774.201836:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879774.201838:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879774.201839:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879774.201841:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290196992:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879774.201844:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879774.201845:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879774.201846:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879774.201847:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879774.201849:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879774.201850:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879774.201852:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290197120:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879774.201854:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879774.201855:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879774.201856:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879774.201857:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879774.201859:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201862:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879774.201862:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:1.0:1788879774.201876:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879774.201882:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879774.201884:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879774.201885:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879774.201886:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201887:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879774.201889:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201891:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305385481 00000100:00000001:1.0:1788879774.201893:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305386025 00000100:00000001:1.0:1788879774.201894:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305385481 00000100:00000001:1.0:1788879774.201895:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305386025 00000100:00000001:1.0:1788879774.201897:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879774.201898:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201899:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879774.201926:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879774/real 1788879774] req@ffff94e50449c000 x1875765290197120/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879774.201932:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879774.201932:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879774.201934:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879774.201935:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879774.201936:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.201937:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879774.201938:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879774.201941:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290197120/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879774.201945:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.201947:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879774.201949:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.201950:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879774.201951:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.201952:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879774.201964:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879774.201966:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879774.201968:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879774.201969:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879774.201971:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290197120:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879774.201974:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.201975:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879774.201976:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879774.201977:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879774.201979:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879774.201979:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879774.201980:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879774.201982:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879774.201983:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879774.201984:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879774.201985:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.201986:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879774.201987:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879774.201988:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879774.201989:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879774.201990:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879774.201991:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879774.201992:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879774.201992:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879774.201993:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879774.201994:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.201995:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.201997:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879774.201998:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.201999:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879774.202000:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879774.202001:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879774.202001:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879774.202002:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879774.202004:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879774.202005:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879774.202006:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879774.202008:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290197248:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879774.202010:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879774.202011:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879774.202012:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879774.202013:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879774.202016:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.202019:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879774.202020:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:1.0:1788879774.202033:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879774.202040:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879774.202043:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879774.202044:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879774.202045:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.202046:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879774.202048:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.202050:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305385482 00000100:00000001:1.0:1788879774.202052:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305386025 00000100:00000001:1.0:1788879774.202053:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305385482 00000100:00000001:1.0:1788879774.202055:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305386025 00000100:00000001:1.0:1788879774.202056:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879774.202058:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.202059:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879774.202061:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879774/real 1788879774] req@ffff94e50449e300 x1875765290197248/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879774.202068:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879774.202070:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879774.202072:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879774.202073:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879774.202075:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.202077:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879774.202078:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879774.202082:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290197248/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879774.202090:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.202092:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879774.202095:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.202096:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879774.202098:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.202100:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879774.202117:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879774.202119:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879774.202122:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879774.202124:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879774.202127:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290197248:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879774.202131:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.202133:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879774.202134:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879774.202135:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879774.202138:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879774.202140:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879774.202141:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879774.202143:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879774.202145:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879774.202147:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879774.202148:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.202151:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.202154:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879774.202155:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.202157:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879774.202159:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879774.264977:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879774.264983:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879774.264984:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879774.264988:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879774.264991:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879774.264995:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.264997:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879774.264999:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879774.265001:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879774.969006:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879774.969011:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879774.969016:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879774.969017:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879774.969155:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879774.969164:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879774.969170:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879775.289024:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879775.289034:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879775.289038:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879775.289048:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879775.289056:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879775.289063:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879775.289066:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879775.289069:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879775.289073:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879775.993041:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879775.993050:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879775.993056:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879775.993061:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879775.993064:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879775.993068:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879775.993071:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879775.993074:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879775.993076:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879775.993079:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879775.993191:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879775.993200:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879775.993206:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879776.313020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879776.313028:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879776.313031:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879776.313038:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879776.313042:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879776.313047:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879776.313051:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879776.313054:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879776.313057:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879777.017048:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879777.017055:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879777.017062:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879777.017064:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879777.017106:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879777.017109:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879777.017112:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879777.336996:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879777.337020:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879777.337023:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879777.337030:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879777.337035:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879777.337040:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879777.337043:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879777.337045:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879777.337048:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879778.040988:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879778.040996:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879778.040998:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879778.041002:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879778.041006:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879778.041008:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879778.041026:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879778.041029:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879778.041033:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879778.041035:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879778.041059:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879778.041061:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879778.041064:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879778.361008:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879778.361017:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879778.361021:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879778.361027:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879778.361032:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879778.361038:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879778.361042:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879778.361045:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879778.361048:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879779.065018:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879779.065026:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879779.065033:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879779.065035:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879779.065127:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879779.065133:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879779.065139:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879779.321030:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879779.321054:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879779.321057:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879779.321062:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879779.321068:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879779.321076:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10718 00000100:00080000:3.0:1788879779.321089:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879779.321092:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879779.321095:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879779.321104:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879779.321107:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879779.321111:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879779.321115:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879779.321117:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879779.321119:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879779.321124:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879779.321127:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879779.321131:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879779.321136:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879779.321139:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879779.321142:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879779.321145:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879779.321159:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879779.321161:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879779.321164:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879779.321168:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879779.321174:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879779.321187:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879779.321198:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290197376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879779.321211:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879779.321216:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879779.321234:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879779.321238:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879779.321242:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879779.321244:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879779.321246:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879779.321250:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879779.321252:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10718 00000100:00080000:3.0:1788879779.321256:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879779.321258:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879779.321260:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879779.321263:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879779.321265:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879779.321268:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879779.321271:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879779.321274:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879779.321277:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879779.321281:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879779.321285:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879779.321288:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879779.321296:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879779.321299:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788879779.321299:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879779.321306:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879779.321307:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879779.321313:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1788879779.321314:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879779.321318:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:3.0:1788879779.321321:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.321323:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879779.321326:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879779.321333:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879779.321334:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879779.321336:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879779.321337:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879779.321339:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879779.321342:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879779.321345:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879779.321349:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879779.321357:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879779.321358:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290197504/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1788879779.321368:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290197376:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879779.321382:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879779.321383:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879779.321390:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879779.321391:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879779.321397:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879779.321401:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879779.321402:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788879779.321407:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879779.321412:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879779.321413:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879779.321419:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879779.321422:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879779.321425:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879779.321430:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10718 00000100:00080000:3.0:1788879779.321433:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879779.321435:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879779.321437:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879779.321438:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1788879779.321442:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879779.321443:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000020:00000001:3.0:1788879779.321447:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879779.321451:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1788879779.321452:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879779.321466:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879779.321490:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879779.321493:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879779.321499:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879779.321502:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879779.321504:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879779.321508:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879779.321512:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879779.321515:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879779.321517:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879779.321520:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879779.321522:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879779.321530:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:3.0:1788879779.321531:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879779.321536:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879779.321539:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879779.321542:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879779.321546:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.321549:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:3.0:1788879779.321554:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.321556:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879779.321562:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.321567:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:3.0:1788879779.321567:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290197632/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879779.321574:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.321584:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305391145 00000100:00000001:3.0:1788879779.321584:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879779.321608:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879779.321613:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879779.321616:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879779.321618:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1788879779.321621:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879779.321627:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879779.321629:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879779.321632:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879779.321636:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10718 00000100:00080000:3.0:1788879779.321641:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879779.321644:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879779.321647:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305391145 00000100:00000001:3.0:1788879779.321648:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879779.321661:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:3.0:1788879779.321661:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879779.321668:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879779.321671:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1788879779.321673:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879779.321678:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879779.321681:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879779.321685:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879779.321688:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879779.321693:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:1.0:1788879779.321700:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879779/real 1788879779] req@ffff94e50449c700 x1875765290197376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1788879779.321705:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879779.321716:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879779.321724:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879779.321731:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:1.0:1788879779.321732:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879779.321739:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879779.321745:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879779.321750:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:3.0:1788879779.321752:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.321753:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879779.321763:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879779.321764:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879779.321773:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1788879779.321773:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879779.321799:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879779.321801:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879779.321806:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879779.321809:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879779.321812:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879779.321815:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1788879779.321818:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290197376/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879779.321819:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879779.321828:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879779.321844:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290197760/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879779.321847:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.321858:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879779.321867:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.321876:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879779.321876:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879779.321884:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879779.321887:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.321891:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879779.321897:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.321898:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879779.321958:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879779.321967:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879779.321970:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879779.321976:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290197376:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879779.321981:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.321984:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879779.321986:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879779.321989:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879779.321994:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879779.321996:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879779.321999:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879779.322003:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879779.322005:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879779.322008:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879779.322009:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322012:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322020:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879779.322023:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.322025:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879779.322027:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879779.322029:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879779.322031:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879779.322033:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879779.322035:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879779.322037:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879779.322040:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879779.322044:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290197504:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879779.322048:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879779.322050:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879779.322052:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879779.322054:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879779.322058:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.322066:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879779.322068:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:1.0:1788879779.322073:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879779.322090:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879779.322109:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879779.322112:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879779.322113:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.322115:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879779.322118:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.322123:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305390604 00000100:00000001:1.0:1788879779.322126:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305391145 00000100:00000001:1.0:1788879779.322129:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305390604 00000100:00000001:1.0:1788879779.322132:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305391145 00000100:00000001:1.0:1788879779.322134:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879779.322136:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.322138:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879779.322143:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879779/real 1788879779] req@ffff94e50449f480 x1875765290197504/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879779.322153:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879779.322154:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879779.322157:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879779.322159:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879779.322161:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322163:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879779.322164:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879779.322169:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290197504/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879779.322185:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322187:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879779.322190:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322192:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879779.322193:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322195:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879779.322214:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879779.322217:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879779.322220:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879779.322222:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879779.322226:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290197504:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879779.322230:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879779.322232:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879779.322233:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879779.322236:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879779.322238:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879779.322240:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879779.322243:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290197632:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879779.322246:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879779.322248:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879779.322249:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879779.322251:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879779.322255:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.322260:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879779.322262:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:1.0:1788879779.322265:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879779.322276:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879779.322280:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879779.322282:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879779.322284:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.322286:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879779.322289:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.322292:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305390602 00000100:00000001:1.0:1788879779.322295:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305391145 00000100:00000001:1.0:1788879779.322297:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305390602 00000100:00000001:1.0:1788879779.322300:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305391145 00000100:00000001:1.0:1788879779.322302:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879779.322304:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.322306:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879779.322310:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879779/real 1788879779] req@ffff94e50449f800 x1875765290197632/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879779.322318:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879779.322319:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879779.322322:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879779.322323:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879779.322325:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322327:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879779.322328:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879779.322332:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290197632/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879779.322339:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322342:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879779.322344:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322345:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879779.322347:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322348:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879779.322366:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879779.322368:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879779.322371:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879779.322373:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879779.322377:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290197632:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879779.322381:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879779.322383:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879779.322384:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879779.322387:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879779.322388:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879779.322390:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879779.322393:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290197760:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879779.322396:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879779.322398:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879779.322399:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879779.322401:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879779.322404:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.322408:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879779.322409:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:1.0:1788879779.322412:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879779.322421:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879779.322425:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879779.322427:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879779.322429:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.322430:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879779.322433:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.322436:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305390602 00000100:00000001:1.0:1788879779.322439:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305391145 00000100:00000001:1.0:1788879779.322441:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305390602 00000100:00000001:1.0:1788879779.322444:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305391145 00000100:00000001:1.0:1788879779.322446:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879779.322448:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.322450:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879779.322453:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879779/real 1788879779] req@ffff94e50449c380 x1875765290197760/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879779.322492:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879779.322494:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879779.322498:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879779.322499:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879779.322501:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322503:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879779.322505:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879779.322510:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290197760/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879779.322518:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322520:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879779.322523:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322525:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879779.322526:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322528:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879779.322544:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879779.322546:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879779.322549:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879779.322551:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879779.322555:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290197760:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879779.322559:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322561:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879779.322563:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879779.322565:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879779.322568:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879779.322569:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879779.322571:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879779.322573:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879779.322575:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879779.322577:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879779.322579:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322581:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879779.322583:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879779.322584:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879779.322586:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879779.322605:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879779.322607:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879779.322609:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879779.322611:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879779.322612:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879779.322614:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322616:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879779.322617:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879779.322618:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879779.322621:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879779.322622:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879779.322623:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879779.322625:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879779.322627:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879779.322628:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879779.322630:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322632:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.322636:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879779.322638:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.322640:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879779.322642:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879779.385013:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879779.385019:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879779.385022:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879779.385027:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879779.385031:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879779.385035:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.385038:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879779.385042:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879779.385045:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879780.089027:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879780.089034:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879780.089040:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879780.089042:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879780.089116:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879780.089121:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879780.089125:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879780.409055:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879780.409066:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879780.409071:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879780.409079:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879780.409085:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879780.409092:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879780.409098:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879780.409103:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879780.409109:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879781.113006:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879781.113014:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879781.113021:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879781.113023:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879781.113048:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879781.113052:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879781.113055:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879781.433051:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879781.433063:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879781.433067:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879781.433076:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879781.433083:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879781.433090:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879781.433095:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879781.433100:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879781.433105:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879782.137081:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879782.137092:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879782.137103:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879782.137123:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879782.137162:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879782.137167:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879782.137172:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879782.457056:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879782.457063:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879782.457066:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879782.457073:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879782.457077:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879782.457082:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879782.457086:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879782.457089:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879782.457092:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879783.161103:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879783.161114:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879783.161121:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879783.161123:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879783.161264:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879783.161272:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879783.161278:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879783.481020:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879783.481027:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879783.481029:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879783.481037:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879783.481043:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879783.481049:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879783.481053:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879783.481058:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879783.481062:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879784.185023:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879784.185031:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879784.185039:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879784.185041:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879784.185123:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879784.185129:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879784.185134:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1788879784.441077:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879784.441095:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1788879784.441100:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879784.441107:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879784.441116:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879784.441127:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10723 00000100:00080000:1.0:1788879784.441145:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879784.441153:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879784.441157:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879784.441169:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879784.441174:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879784.441179:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:1.0:1788879784.441184:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:1.0:1788879784.441187:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879784.441190:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879784.441198:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879784.441204:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879784.441210:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879784.441216:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879784.441222:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879784.441227:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879784.441231:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879784.441248:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879784.441250:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879784.441254:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879784.441258:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879784.441263:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879784.441277:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879784.441300:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e51100a680 x1875765290197888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879784.441315:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879784.441321:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879784.441343:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879784.441348:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879784.441353:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1788879784.441355:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879784.441357:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879784.441361:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879784.441364:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10723 00000100:00080000:1.0:1788879784.441371:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879784.441374:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879784.441376:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879784.441381:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879784.441384:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879784.441388:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:1.0:1788879784.441393:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879784.441397:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879784.441400:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879784.441406:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879784.441411:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879784.441415:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879784.441424:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879784.441430:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879784.441469:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879784.441476:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:1.0:1788879784.441477:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879784.441486:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.441488:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879784.441490:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.441498:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1788879784.441501:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879784.441506:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1788879784.441507:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879784.441514:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879784.441516:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879784.441518:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879784.441523:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879784.441529:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1788879784.441533:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879784.441539:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1788879784.441547:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009880 x1875765290198016/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788879784.441550:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879784.441568:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290197888:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879784.441569:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879784.441574:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879784.441578:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879784.441582:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879784.441588:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879784.441603:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1788879784.441605:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879784.441611:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1788879784.441613:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1788879784.441616:0:178134:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:2.0:1788879784.441619:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879784.441626:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10723 02000000:00000001:2.0:1788879784.441628:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1788879784.441638:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1788879784.441641:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879784.441643:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879784.441649:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:1.0:1788879784.441655:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879784.441657:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1788879784.441660:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879784.441663:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879784.441666:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879784.441669:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879784.441670:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879784.441676:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286517888 : -117763423033728 : ffff94e51100a680) 00000100:00080000:1.0:1788879784.441678:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879784.441686:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879784.441687:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879784.441695:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879784.441701:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879784.441706:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879784.441711:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879784.441715:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879784.441721:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879784.441726:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879784.441735:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879784.441739:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788879784.441746:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879784.441749:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879784.441754:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879784.441757:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879784.441767:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.441768:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879784.441777:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1788879784.441780:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511009180 x1875765290198144/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879784.441782:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.441792:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879784.441801:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879784.441803:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879784.441809:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.441810:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305396265 00000100:00000001:1.0:1788879784.441817:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879784.441823:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1788879784.441827:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879784.441828:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788879784.441830:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1788879784.441836:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879784.441841:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305396265 00000100:00000001:1.0:1788879784.441842:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:1.0:1788879784.441851:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10723 00000100:00000001:2.0:1788879784.441853:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1788879784.441860:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879784.441861:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879784.441870:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879784.441871:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879784.441876:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879784.441878:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:1.0:1788879784.441886:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1788879784.441891:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:2.0:1788879784.441895:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879784/real 1788879784] req@ffff94e51100a680 x1875765290197888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1788879784.441896:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:1.0:1788879784.441947:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879784.441951:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879784.441953:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:1.0:1788879784.441960:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879784.441965:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879784.441973:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879784.441981:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879784.441999:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879784.442002:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:1.0:1788879784.442005:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1788879784.442013:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879784.442016:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1788879784.442019:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879784.442021:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879784.442034:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:1.0:1788879784.442035:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1788879784.442038:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:1.0:1788879784.442042:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.442044:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879784.442053:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1788879784.442054:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1788879784.442063:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879784.442063:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1788879784.442073:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879784.442077:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e51100a680 x1875765290197888/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879784.442099:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.442101:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442113:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879784.442120:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879784.442124:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e511008380 x1875765290198272/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879784.442124:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879784.442129:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442136:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879784.442141:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879784.442152:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1788879784.442164:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00080000:2.0:1788879784.442168:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879784.442172:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.442175:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1788879784.442179:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879784.442187:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e51100a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290197888:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879784.442197:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442200:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879784.442202:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879784.442205:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879784.442210:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879784.442213:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879784.442216:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879784.442221:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879784.442224:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879784.442228:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879784.442230:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442232:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442241:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879784.442244:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.442247:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879784.442249:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879784.442251:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879784.442253:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879784.442255:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879784.442258:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879784.442261:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879784.442263:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879784.442267:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290198016:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879784.442272:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879784.442274:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879784.442276:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879784.442278:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879784.442282:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.442289:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879784.442291:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286514304 : -117763423037312 : ffff94e511009880) 00000100:00000001:2.0:1788879784.442298:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879784.442314:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879784.442320:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879784.442323:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879784.442325:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.442327:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879784.442331:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.442336:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305395724 00000100:00000001:2.0:1788879784.442340:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305396265 00000100:00000001:2.0:1788879784.442343:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305395724 00000100:00000001:2.0:1788879784.442346:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305396265 00000100:00000001:2.0:1788879784.442349:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879784.442351:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.442353:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879784.442358:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879784/real 1788879784] req@ffff94e511009880 x1875765290198016/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879784.442369:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879784.442370:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879784.442373:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879784.442375:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879784.442377:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442380:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879784.442381:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879784.442386:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009880 x1875765290198016/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879784.442395:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442399:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879784.442403:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442406:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879784.442408:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442412:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879784.442468:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879784.442477:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879784.442484:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879784.442490:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879784.442497:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290198016:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879784.442505:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879784.442508:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879784.442511:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879784.442515:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879784.442518:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879784.442522:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879784.442528:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290198144:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879784.442533:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879784.442536:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879784.442538:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879784.442541:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879784.442546:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.442555:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879784.442557:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286512512 : -117763423039104 : ffff94e511009180) 00000100:00000001:2.0:1788879784.442563:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879784.442581:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879784.442587:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879784.442590:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879784.442593:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.442595:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879784.442599:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.442603:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305395722 00000100:00000001:2.0:1788879784.442607:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305396265 00000100:00000001:2.0:1788879784.442610:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305395722 00000100:00000001:2.0:1788879784.442613:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305396265 00000100:00000001:2.0:1788879784.442615:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879784.442618:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.442621:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879784.442626:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879784/real 1788879784] req@ffff94e511009180 x1875765290198144/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879784.442637:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879784.442639:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879784.442642:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879784.442644:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879784.442647:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442649:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879784.442651:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879784.442655:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511009180 x1875765290198144/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879784.442664:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442667:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879784.442669:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442671:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879784.442673:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442675:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879784.442693:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879784.442696:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879784.442700:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879784.442702:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879784.442706:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511009180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290198144:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879784.442710:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879784.442712:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879784.442714:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879784.442717:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879784.442719:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879784.442722:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879784.442725:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290198272:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879784.442732:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879784.442734:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879784.442736:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879784.442748:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879784.442752:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.442757:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879784.442759:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310286508928 : -117763423042688 : ffff94e511008380) 00000100:00000001:2.0:1788879784.442763:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879784.442774:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879784.442779:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879784.442781:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879784.442783:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.442785:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879784.442788:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.442792:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305395722 00000100:00000001:2.0:1788879784.442795:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305396265 00000100:00000001:2.0:1788879784.442798:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305395722 00000100:00000001:2.0:1788879784.442800:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305396265 00000100:00000001:2.0:1788879784.442815:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879784.442817:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.442819:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879784.442823:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879784/real 1788879784] req@ffff94e511008380 x1875765290198272/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879784.442832:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879784.442833:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879784.442836:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879784.442838:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879784.442839:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442841:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879784.442843:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879784.442847:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e511008380 x1875765290198272/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879784.442863:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442866:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879784.442869:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442871:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879784.442872:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442874:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879784.442892:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879784.442895:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879784.442898:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879784.442929:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879784.442935:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e511008380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290198272:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879784.442940:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442942:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879784.442944:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879784.442946:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879784.442949:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879784.442951:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879784.442953:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879784.442956:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879784.442957:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879784.442960:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879784.442962:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442964:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879784.442966:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879784.442967:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879784.442969:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879784.442971:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879784.442972:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879784.442974:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879784.442975:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879784.442977:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879784.442979:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.442981:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879784.442982:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879784.442989:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879784.442991:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879784.442993:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879784.442995:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879784.442997:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879784.442998:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879784.443000:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879784.443002:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.443004:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879784.443009:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879784.443010:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879784.443013:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879784.443014:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879784.505032:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879784.505040:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879784.505044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879784.505052:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879784.505058:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879784.505064:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879784.505069:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879784.505073:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879784.505077:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879785.209027:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879785.209038:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879785.209048:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879785.209054:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879785.209058:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879785.209064:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879785.209070:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879785.209074:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879785.209078:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879785.209083:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879785.209128:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879785.209132:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879785.209136:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879785.529013:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879785.529023:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879785.529027:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879785.529035:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879785.529040:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879785.529046:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879785.529049:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879785.529053:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879785.529057:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879786.233009:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879786.233014:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879786.233018:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879786.233019:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879786.233038:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879786.233041:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879786.233043:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879786.552941:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879786.552948:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879786.552949:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879786.552952:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879786.552955:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879786.552958:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879786.552960:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879786.552962:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879786.552964:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879787.257056:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879787.257060:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879787.257065:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879787.257066:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879787.257145:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879787.257152:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879787.257158:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879787.577048:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879787.577053:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879787.577055:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879787.577059:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879787.577062:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879787.577065:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879787.577068:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879787.577070:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879787.577073:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879788.281094:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879788.281104:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879788.281108:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879788.281112:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879788.281119:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879788.281122:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879788.281125:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879788.281128:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879788.281133:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879788.281135:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879788.281212:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879788.281219:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879788.281224:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879788.600996:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879788.601008:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879788.601013:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879788.601023:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879788.601030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879788.601037:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879788.601042:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879788.601047:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879788.601053:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879789.305056:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879789.305064:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879789.305071:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879789.305073:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879789.305137:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879789.305140:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879789.305144:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879789.561095:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879789.561112:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879789.561117:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879789.561125:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879789.561133:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879789.561146:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10728 00000100:00080000:3.0:1788879789.561164:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879789.561171:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879789.561176:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879789.561189:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879789.561194:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879789.561198:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879789.561202:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879789.561205:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879789.561207:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879789.561214:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879789.561219:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879789.561223:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879789.561228:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879789.561232:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879789.561235:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879789.561238:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879789.561252:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879789.561254:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879789.561258:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879789.561262:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879789.561269:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879789.561284:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879789.561296:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290198400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879789.561311:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879789.561318:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879789.561341:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879789.561346:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879789.561351:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879789.561353:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879789.561356:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879789.561359:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879789.561363:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10728 00000100:00080000:3.0:1788879789.561369:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879789.561372:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879789.561374:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879789.561379:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879789.561381:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879789.561384:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879789.561386:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879789.561389:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879789.561391:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879789.561395:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879789.561425:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879789.561430:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879789.561440:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879789.561446:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879789.561451:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879789.561458:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879789.561459:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879789.561468:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00000001:3.0:1788879789.561469:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.561473:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879789.561480:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.561481:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879789.561487:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879789.561496:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879789.561496:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879789.561507:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879789.561508:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879789.561520:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879789.561521:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879789.561529:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1788879789.561531:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290198400:192.168.203.129@tcp:250:kworker.0 02000000:00000001:3.0:1788879789.561538:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879789.561549:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.561560:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1788879789.561569:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290198528/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1788879789.561570:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879789.561577:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879789.561586:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879789.561595:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1788879789.561597:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879789.561601:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879789.561609:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879789.561613:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879789.561618:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879789.561621:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1788879789.561623:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1788879789.561625:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879789.561630:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:3.0:1788879789.561633:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:2.0:1788879789.561640:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879789.561641:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10728 00000100:00080000:3.0:1788879789.561651:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879789.561654:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879789.561656:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879789.561678:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879789.561681:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879789.561684:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879789.561688:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879789.561691:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879789.561696:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000800:00000100:2.0:1788879789.561698:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1788879789.561708:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879789.561714:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879789.561718:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879789.561719:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788879789.561723:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879789.561729:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788879789.561730:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.561737:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879789.561738:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879789.561745:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879789.561746:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000008:00000001:3.0:1788879789.561754:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.561757:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879789.561758:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.561770:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305401385 02000000:00000001:3.0:1788879789.561774:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879789.561780:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879789.561783:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879789.561785:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879789.561787:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.561789:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1788879789.561797:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879789.561807:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290198656/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879789.561816:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305401385 00000100:00000001:3.0:1788879789.561819:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879789.561833:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879789.561834:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879789.561846:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.561850:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879789.561851:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879789.561865:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879789.561867:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879789.561869:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879789.561875:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879789.561879:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879789.561882:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879789.561892:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10728 00000100:00080000:3.0:1788879789.561897:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879789.561946:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879789.561948:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879789.561954:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879789.561956:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879789.561960:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00100000:2.0:1788879789.561961:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879789/real 1788879789] req@ffff94e50449d880 x1875765290198400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788879789.561967:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879789.561976:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879789.561981:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879789.561995:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000400:00000001:2.0:1788879789.562001:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788879789.562001:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879789.562009:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1788879789.562011:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879789.562021:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879789.562024:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:3.0:1788879789.562025:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562033:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879789.562033:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879789.562037:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879789.562039:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1788879789.562044:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562047:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879789.562051:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562053:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879789.562063:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00100000:2.0:1788879789.562069:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290198400/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879789.562070:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879789.562082:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879789.562089:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879789.562098:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879789.562098:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562111:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879789.562119:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879789.562119:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562127:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879789.562132:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562136:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879789.562141:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:3.0:1788879789.562145:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290198784/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1788879789.562147:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879789.562161:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879789.562167:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879789.562181:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290198400:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879789.562189:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879789.562192:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879789.562198:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562201:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879789.562208:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1788879789.562209:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879789.562213:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879789.562219:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879789.562223:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879789.562225:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879789.562231:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879789.562233:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879789.562236:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879789.562238:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562242:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562252:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879789.562256:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562260:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879789.562262:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879789.562264:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879789.562266:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879789.562268:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879789.562271:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879789.562274:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879789.562277:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879789.562281:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290198528:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879789.562286:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879789.562289:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879789.562291:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879789.562293:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879789.562297:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562305:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879789.562307:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:2.0:1788879789.562313:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879789.562332:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879789.562339:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879789.562342:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879789.562344:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562347:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879789.562350:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562355:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305400844 00000100:00000001:2.0:1788879789.562359:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305401385 00000100:00000001:2.0:1788879789.562362:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305400844 00000100:00000001:2.0:1788879789.562365:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305401385 00000100:00000001:2.0:1788879789.562368:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879789.562370:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562372:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879789.562378:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879789/real 1788879789] req@ffff94e50449fb80 x1875765290198528/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879789.562388:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879789.562389:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879789.562392:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879789.562394:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879789.562418:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562422:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879789.562424:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879789.562430:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290198528/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879789.562439:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562442:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879789.562445:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562447:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879789.562449:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562451:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879789.562471:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879789.562474:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879789.562477:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879789.562480:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879789.562485:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290198528:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879789.562490:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879789.562492:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879789.562494:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879789.562496:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879789.562499:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879789.562501:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879789.562505:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290198656:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879789.562509:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879789.562511:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879789.562514:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879789.562516:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879789.562520:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562527:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879789.562529:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:2.0:1788879789.562535:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879789.562550:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879789.562556:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879789.562559:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879789.562561:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562563:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879789.562566:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562570:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305400842 00000100:00000001:2.0:1788879789.562574:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305401385 00000100:00000001:2.0:1788879789.562577:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305400842 00000100:00000001:2.0:1788879789.562580:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305401385 00000100:00000001:2.0:1788879789.562582:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879789.562585:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562587:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879789.562591:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879789/real 1788879789] req@ffff94e50449d180 x1875765290198656/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879789.562609:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879789.562611:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879789.562614:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879789.562616:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879789.562618:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562620:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879789.562622:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879789.562626:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290198656/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879789.562635:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562637:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879789.562640:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562642:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879789.562643:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562645:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879789.562663:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879789.562666:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879789.562669:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879789.562672:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879789.562675:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290198656:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879789.562679:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879789.562681:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879789.562683:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879789.562685:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879789.562687:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879789.562690:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879789.562693:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290198784:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879789.562697:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879789.562699:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879789.562702:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879789.562704:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879789.562707:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562712:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879789.562714:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:2.0:1788879789.562718:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879789.562729:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879789.562733:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879789.562736:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879789.562738:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562740:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879789.562743:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562746:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305400842 00000100:00000001:2.0:1788879789.562750:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305401385 00000100:00000001:2.0:1788879789.562753:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305400842 00000100:00000001:2.0:1788879789.562756:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305401385 00000100:00000001:2.0:1788879789.562758:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879789.562761:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562763:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879789.562767:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879789/real 1788879789] req@ffff94e50449ce00 x1875765290198784/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879789.562776:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879789.562777:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879789.562779:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879789.562781:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879789.562783:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562785:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879789.562787:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879789.562792:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290198784/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879789.562800:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562803:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879789.562806:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562808:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879789.562810:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562812:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879789.562834:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879789.562840:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879789.562846:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879789.562850:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879789.562857:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290198784:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879789.562865:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562869:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879789.562872:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879789.562874:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879789.562879:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879789.562881:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879789.562884:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879789.562888:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879789.562891:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879789.562895:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879789.562898:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562901:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879789.562936:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879789.562939:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879789.562942:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879789.562944:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879789.562946:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879789.562949:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879789.562950:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879789.562953:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879789.562955:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562957:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879789.562959:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879789.562960:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879789.562963:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879789.562965:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879789.562966:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879789.562968:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879789.562971:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879789.562973:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879789.562975:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562977:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879789.562982:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879789.562984:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879789.562986:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879789.562988:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879789.624998:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879789.625004:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879789.625007:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879789.625013:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879789.625016:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879789.625020:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879789.625023:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879789.625025:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879789.625028:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879790.329057:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879790.329064:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879790.329071:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879790.329073:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879790.329101:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879790.329105:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879790.329108:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879790.648998:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879790.649005:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879790.649007:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879790.649020:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879790.649025:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879790.649030:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879790.649033:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879790.649037:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879790.649040:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879791.353018:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879791.353027:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879791.353034:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879791.353036:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879791.353132:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879791.353138:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879791.353145:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879791.673015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879791.673026:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879791.673029:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879791.673035:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879791.673040:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879791.673045:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879791.673049:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879791.673052:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879791.673056:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879792.377041:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879792.377050:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879792.377058:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879792.377060:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879792.377155:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879792.377161:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879792.377166:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879792.696994:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879792.697002:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879792.697005:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879792.697011:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879792.697014:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879792.697019:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879792.697038:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879792.697041:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879792.697044:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879793.401019:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879793.401027:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879793.401032:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879793.401033:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879793.401071:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879793.401073:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879793.401075:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879793.720993:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879793.720999:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879793.721001:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879793.721004:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879793.721007:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879793.721011:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879793.721013:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879793.721016:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879793.721018:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879794.425027:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879794.425034:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879794.425041:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879794.425042:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879794.425066:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879794.425068:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879794.425071:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1788879794.681126:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879794.681141:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1788879794.681144:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879794.681149:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879794.681155:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879794.681165:0:178195:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10733 00000100:00080000:0.0:1788879794.681180:0:178195:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879794.681183:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879794.681186:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:0.0:1788879794.681197:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879794.681201:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879794.681206:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:0.0:1788879794.681209:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:0.0:1788879794.681212:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879794.681213:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:0.0:1788879794.681219:0:178195:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879794.681222:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879794.681225:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879794.681230:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879794.681234:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879794.681237:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:0.0:1788879794.681240:0:178195:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879794.681257:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879794.681259:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879794.681264:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879794.681267:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879794.681274:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879794.681290:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1788879794.681302:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290198912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879794.681315:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879794.681322:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879794.681345:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879794.681350:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879794.681354:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1788879794.681356:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879794.681358:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879794.681384:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879794.681407:0:178195:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10733 00000100:00000001:2.0:1788879794.681407:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1788879794.681422:0:178195:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879794.681423:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879794.681430:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879794.681431:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879794.681437:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879794.681437:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879794.681446:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879794.681449:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1788879794.681458:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1788879794.681463:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 02000000:00000001:2.0:1788879794.681463:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1788879794.681469:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:0.0:1788879794.681471:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879794.681474:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879794.681480:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1788879794.681481:0:178195:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:0.0:1788879794.681489:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879794.681494:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879794.681495:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879794.681504:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879794.681510:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879794.681512:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290198912:192.168.203.129@tcp:250:kworker.0 00000100:00000001:0.0:1788879794.681515:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879794.681524:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:0.0:1788879794.681525:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1788879794.681531:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879794.681531:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879794.681537:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879794.681538:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879794.681549:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879794.681552:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879794.681555:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879794.681558:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879794.681560:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1788879794.681564:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879794.681564:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879794.681575:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.681585:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00100000:0.0:1788879794.681586:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508979180 x1875765290199040/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879794.681589:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000100:00000001:2.0:1788879794.681603:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1788879794.681613:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879794.681617:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879794.681623:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879794.681627:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879794.681633:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1788879794.681638:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879794.681640:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879794.681663:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879794.681671:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10733 00000800:00000100:2.0:1788879794.681672:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:0.0:1788879794.681682:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1788879794.681689:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1788879794.681692:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879794.681696:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:0.0:1788879794.681700:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879794.681705:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1788879794.681706:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879794.681712:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879794.681713:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879794.681720:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:0.0:1788879794.681723:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879794.681730:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879794.681732:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879794.681737:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305406505 00000100:00000001:0.0:1788879794.681740:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879794.681750:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1788879794.681753:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879794.681759:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305406505 00000100:00000001:0.0:1788879794.681761:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.681766:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1788879794.681769:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1788879794.681771:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879794.681778:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879794.681781:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879794.681781:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1788879794.681792:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879794.681796:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879794.681818:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1788879794.681821:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879794/real 1788879794] req@ffff94e50897b800 x1875765290198912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1788879794.681823:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879794.681836:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879794.681852:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1788879794.681855:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1788879794.681857:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1788879794.681860:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:2.0:1788879794.681860:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1788879794.681867:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879794.681870:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:0.0:1788879794.681872:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.681876:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879794.681879:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879794.681882:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.681884:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879794.681893:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1788879794.681901:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e508978380 x1875765290199168/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879794.681913:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879794.681916:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879794.681920:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1788879794.681923:0:178195:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1788879794.681929:0:178195:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:0.0:1788879794.681931:0:178195:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1788879794.681935:0:178195:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1788879794.681939:0:178195:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:0.0:1788879794.681944:0:178195:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10733 00000100:00100000:2.0:1788879794.681945:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290198912/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879794.681959:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.681965:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879794.681969:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.681972:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879794.681984:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.681987:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879794.681989:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879794.681994:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879794.682001:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879794.682006:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879794.682011:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290198912:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879794.682017:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682020:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879794.682022:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879794.682032:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:0.0:1788879794.682036:0:178195:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1788879794.682037:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879794.682041:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:0.0:1788879794.682047:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1788879794.682048:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:0.0:1788879794.682057:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:2.0:1788879794.682059:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879794.682062:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879794.682069:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000020:00000001:0.0:1788879794.682072:0:178195:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1788879794.682073:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788879794.682080:0:178195:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1788879794.682081:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1788879794.682092:0:178195:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879794.682095:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879794.682098:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879794.682101:0:178195:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879794.682103:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879794.682107:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879794.682109:0:178195:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879794.682111:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1788879794.682113:0:178195:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:2.0:1788879794.682115:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879794.682119:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879794.682134:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1788879794.682142:0:178195:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:2.0:1788879794.682142:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1788879794.682150:0:178195:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879794.682154:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879794.682155:0:178195:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1788879794.682161:0:178195:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1788879794.682169:0:178195:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879794.682170:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290199040:192.168.203.129@tcp:38:kworker.0 00000100:00000001:0.0:1788879794.682174:0:178195:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:0.0:1788879794.682180:0:178195:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1788879794.682183:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1788879794.682188:0:178195:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879794.682191:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1788879794.682193:0:178195:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879794.682195:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879794.682199:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1788879794.682203:0:178195:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879794.682203:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1788879794.682210:0:178195:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879794.682214:0:178195:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1788879794.682217:0:178195:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1788879794.682217:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879794.682221:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145413504 : -117763564138112 : ffff94e508979180) 02000000:00000001:0.0:1788879794.682222:0:178195:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879794.682234:0:178195:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.682241:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1788879794.682244:0:178195:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50897b800 x1875765290199296/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1788879794.682256:0:178195:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1788879794.682259:0:178195:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879794.682265:0:178195:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1788879794.682273:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879794.682283:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879794.682287:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879794.682288:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.682289:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879794.682293:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.682298:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305405964 00000100:00000001:2.0:1788879794.682301:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305406505 00000100:00000001:2.0:1788879794.682304:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305405964 00000100:00000001:2.0:1788879794.682306:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305406505 00000100:00000001:2.0:1788879794.682307:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879794.682309:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.682311:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879794.682315:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879794/real 1788879794] req@ffff94e508979180 x1875765290199040/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879794.682322:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879794.682323:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879794.682325:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879794.682326:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879794.682327:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682328:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879794.682329:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879794.682332:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508979180 x1875765290199040/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879794.682336:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682338:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879794.682339:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682340:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879794.682341:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682342:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879794.682357:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879794.682359:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879794.682375:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879794.682377:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879794.682379:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508979180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290199040:192.168.203.129@tcp:38:kworker.0 00000100:00000001:2.0:1788879794.682390:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879794.682391:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879794.682392:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879794.682394:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879794.682397:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879794.682398:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879794.682400:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290199168:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879794.682403:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879794.682405:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879794.682406:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879794.682407:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879794.682410:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.682415:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879794.682416:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145409920 : -117763564141696 : ffff94e508978380) 00000100:00000001:2.0:1788879794.682419:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879794.682429:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879794.682432:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879794.682434:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879794.682435:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.682436:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879794.682437:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.682440:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305405962 00000100:00000001:2.0:1788879794.682442:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305406505 00000100:00000001:2.0:1788879794.682443:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305405962 00000100:00000001:2.0:1788879794.682456:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305406505 00000100:00000001:2.0:1788879794.682458:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879794.682460:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.682462:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879794.682466:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879794/real 1788879794] req@ffff94e508978380 x1875765290199168/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879794.682475:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879794.682476:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879794.682479:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879794.682481:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879794.682482:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682484:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879794.682486:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879794.682490:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e508978380 x1875765290199168/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879794.682499:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682501:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879794.682503:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682505:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879794.682507:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682508:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879794.682524:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879794.682525:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879794.682528:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879794.682530:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879794.682532:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e508978380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290199168:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879794.682535:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682537:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879794.682538:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879794.682539:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879794.682541:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879794.682542:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879794.682543:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879794.682545:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879794.682546:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879794.682548:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879794.682549:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682551:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879794.682551:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879794.682552:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879794.682553:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879794.682554:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879794.682555:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879794.682556:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879794.682556:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879794.682557:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879794.682558:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682559:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682563:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879794.682564:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.682566:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879794.682567:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879794.682568:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879794.682569:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1788879794.682571:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879794.682573:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879794.682574:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879794.682575:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879794.682578:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290199296:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879794.682580:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879794.682582:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879794.682583:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879794.682584:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1788879794.682587:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.682591:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879794.682592:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310145423360 : -117763564128256 : ffff94e50897b800) 00000100:00000001:2.0:1788879794.682596:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1788879794.682605:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879794.682608:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879794.682610:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879794.682611:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.682612:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:2.0:1788879794.682614:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.682617:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305405962 00000100:00000001:2.0:1788879794.682619:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305406505 00000100:00000001:2.0:1788879794.682621:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305405962 00000100:00000001:2.0:1788879794.682622:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305406505 00000100:00000001:2.0:1788879794.682624:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879794.682625:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.682626:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1788879794.682629:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879794/real 1788879794] req@ffff94e50897b800 x1875765290199296/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1788879794.682639:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1788879794.682640:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879794.682642:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:2.0:1788879794.682643:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:2.0:1788879794.682644:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682645:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879794.682647:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879794.682649:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50897b800 x1875765290199296/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879794.682654:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682656:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879794.682657:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682659:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879794.682660:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682661:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1788879794.682678:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1788879794.682681:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1788879794.682685:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879794.682688:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1788879794.682691:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50897b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290199296:192.168.203.129@tcp:8:kworker.0 00000100:00000001:2.0:1788879794.682696:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682699:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879794.682700:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1788879794.682702:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1788879794.682705:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1788879794.682706:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:2.0:1788879794.682708:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:2.0:1788879794.682710:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1788879794.682711:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1788879794.682713:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1788879794.682714:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682727:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879794.682729:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1788879794.682730:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879794.682732:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1788879794.682734:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879794.744969:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879794.744977:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879794.744980:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879794.744986:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879794.744990:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879794.745002:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879794.745006:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879794.745008:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879794.745010:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879795.448980:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879795.448987:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879795.448991:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879795.448994:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879795.448996:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879795.448999:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879795.449002:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879795.449004:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879795.449006:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879795.449008:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879795.449048:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879795.449051:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879795.449054:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879795.769042:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879795.769055:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879795.769059:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879795.769070:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879795.769079:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879795.769085:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879795.769090:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879795.769095:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879795.769101:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879796.473117:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879796.473126:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879796.473134:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879796.473136:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879796.473237:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879796.473244:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879796.473249:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879796.793022:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879796.793030:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879796.793033:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879796.793039:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879796.793043:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879796.793048:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879796.793051:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879796.793054:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879796.793057:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879797.497007:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879797.497012:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879797.497017:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879797.497018:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879797.497056:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879797.497059:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879797.497061:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879797.817005:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879797.817012:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879797.817014:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879797.817019:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879797.817023:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879797.817027:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879797.817030:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879797.817033:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879797.817036:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879798.521006:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879798.521013:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879798.521015:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879798.521018:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879798.521022:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879798.521024:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879798.521026:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879798.521027:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879798.521030:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879798.521032:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879798.521062:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879798.521064:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879798.521067:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879798.841014:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879798.841021:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879798.841024:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879798.841028:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879798.841030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879798.841034:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879798.841036:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879798.841039:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879798.841041:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879799.545033:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879799.545039:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879799.545042:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879799.545043:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879799.545121:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879799.545125:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879799.545130:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879799.801101:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879799.801117:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879799.801120:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879799.801126:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879799.801148:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879799.801155:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10738 00000100:00080000:2.0:1788879799.801169:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879799.801172:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879799.801174:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879799.801185:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879799.801188:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879799.801192:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879799.801196:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879799.801198:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879799.801200:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879799.801205:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879799.801209:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879799.801212:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879799.801217:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879799.801220:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801222:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879799.801224:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879799.801239:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879799.801241:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801244:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879799.801246:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879799.801251:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801265:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879799.801277:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290199424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879799.801289:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879799.801295:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801320:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879799.801341:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879799.801345:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879799.801346:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879799.801348:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879799.801350:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879799.801353:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10738 00000100:00080000:2.0:1788879799.801357:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879799.801358:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879799.801359:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879799.801373:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879799.801375:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879799.801377:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879799.801379:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879799.801381:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879799.801382:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879799.801387:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879799.801390:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879799.801391:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879799.801398:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879799.801400:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801402:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879799.801406:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:2.0:1788879799.801407:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879799.801416:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000008:00000001:2.0:1788879799.801416:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801424:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879799.801439:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879799.801440:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801442:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879799.801443:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879799.801444:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.801446:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1788879799.801450:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.801451:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879799.801457:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879799.801463:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290199552/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879799.801465:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879799.801472:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879799.801482:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879799.801483:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801486:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801490:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879799.801493:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290199424:192.168.203.129@tcp:250:kworker.0 00000100:00080000:2.0:1788879799.801494:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879799.801513:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1788879799.801513:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879799.801523:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1788879799.801526:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1788879799.801526:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1788879799.801532:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801533:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879799.801539:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10738 02000000:00000001:1.0:1788879799.801540:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1788879799.801548:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879799.801550:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879799.801551:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879799.801553:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879799.801555:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879799.801562:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879799.801565:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879799.801567:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879799.801570:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879799.801572:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879799.801576:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879799.801581:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1788879799.801581:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879799.801586:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879799.801587:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 02000000:00000001:2.0:1788879799.801593:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879799.801595:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1788879799.801600:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801604:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879799.801607:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879799.801611:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801613:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879799.801619:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879799.801620:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801623:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879799.801625:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879799.801627:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801634:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879799.801639:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290199680/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788879799.801650:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:2.0:1788879799.801674:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879799.801679:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.801681:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879799.801691:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.801693:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1788879799.801697:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879799.801698:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.801707:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00080000:2.0:1788879799.801718:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879799.801724:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801725:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879799.801733:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879799.801735:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305411625 00000100:00000001:2.0:1788879799.801741:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879799.801747:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10738 00000100:00000001:1.0:1788879799.801750:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1788879799.801755:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879799.801759:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879799.801775:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305411625 00000100:00000001:2.0:1788879799.801781:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879799.801784:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1788879799.801788:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879799.801791:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1788879799.801792:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879799.801795:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879799.801797:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879799.801800:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000020:00000001:2.0:1788879799.801802:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879799.801807:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879799.801812:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:1.0:1788879799.801814:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879799/real 1788879799] req@ffff94e5045e4700 x1875765290199424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879799.801846:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1788879799.801847:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000400:00000001:1.0:1788879799.801852:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879799.801853:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.801859:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:2.0:1788879799.801861:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879799.801871:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:2.0:1788879799.801873:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879799.801875:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879799.801877:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.801880:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1788879799.801883:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879799.801884:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1788879799.801889:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879799.801895:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290199424/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:2.0:1788879799.801897:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801905:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879799.801928:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879799.801929:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801932:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879799.801933:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879799.801935:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.801941:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879799.801948:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290199808/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879799.801966:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.801970:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879799.801973:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.801974:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879799.801975:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.801977:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879799.801979:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879799.801993:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879799.801998:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879799.801999:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879799.802003:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290199424:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879799.802007:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802009:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879799.802011:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879799.802014:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879799.802018:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879799.802021:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879799.802023:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879799.802027:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879799.802029:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879799.802033:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879799.802035:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802038:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802045:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879799.802058:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.802060:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879799.802061:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879799.802062:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879799.802066:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879799.802073:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879799.802073:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879799.802079:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1788879799.802080:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879799.802085:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1788879799.802088:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879799.802089:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879799.802105:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290199552:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879799.802108:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879799.802111:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879799.802113:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879799.802114:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879799.802117:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.802122:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879799.802124:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00000001:1.0:1788879799.802128:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879799.802152:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879799.802155:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879799.802158:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879799.802159:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.802160:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879799.802161:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.802165:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305411084 00000100:00000001:1.0:1788879799.802167:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305411625 00000100:00000001:1.0:1788879799.802169:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305411084 00000100:00000001:1.0:1788879799.802171:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305411625 00000100:00000001:1.0:1788879799.802172:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879799.802173:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.802174:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879799.802177:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879799/real 1788879799] req@ffff94e5045e7480 x1875765290199552/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879799.802182:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879799.802183:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879799.802185:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879799.802186:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879799.802187:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802188:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879799.802189:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879799.802191:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290199552/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879799.802195:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802197:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879799.802199:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802200:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879799.802201:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802202:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879799.802226:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879799.802228:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879799.802231:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879799.802232:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879799.802234:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290199552:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879799.802236:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879799.802238:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879799.802239:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879799.802240:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879799.802241:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879799.802242:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879799.802244:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290199680:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879799.802247:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879799.802248:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879799.802249:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879799.802250:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879799.802252:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.802255:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879799.802256:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074559872 : -117763634991744 : ffff94e5045e6d80) 00000100:00000001:1.0:1788879799.802258:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879799.802264:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879799.802266:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879799.802267:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879799.802268:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.802269:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879799.802271:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.802273:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305411082 00000100:00000001:1.0:1788879799.802286:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305411625 00000100:00000001:1.0:1788879799.802288:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305411082 00000100:00000001:1.0:1788879799.802289:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305411625 00000100:00000001:1.0:1788879799.802291:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879799.802292:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.802293:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879799.802295:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879799/real 1788879799] req@ffff94e5045e6d80 x1875765290199680/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879799.802300:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879799.802301:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879799.802302:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879799.802303:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879799.802304:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802305:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879799.802305:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879799.802307:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6d80 x1875765290199680/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879799.802312:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802313:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879799.802315:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802316:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879799.802317:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802318:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879799.802347:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879799.802349:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879799.802351:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879799.802353:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879799.802355:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290199680:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879799.802358:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802359:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879799.802360:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879799.802362:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879799.802364:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879799.802365:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879799.802366:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879799.802368:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879799.802369:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879799.802370:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879799.802371:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802372:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879799.802373:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879799.802374:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879799.802376:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879799.802376:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879799.802377:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879799.802378:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879799.802378:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879799.802380:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879799.802380:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802382:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802384:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879799.802385:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.802386:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879799.802387:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879799.802388:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879799.802389:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879799.802390:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879799.802391:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879799.802392:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879799.802394:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879799.802396:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290199808:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879799.802398:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879799.802400:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879799.802401:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879799.802402:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879799.802404:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.802408:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879799.802409:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:1.0:1788879799.802412:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879799.802420:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879799.802423:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879799.802425:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879799.802426:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.802427:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879799.802429:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.802431:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305411082 00000100:00000001:1.0:1788879799.802433:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305411625 00000100:00000001:1.0:1788879799.802435:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305411082 00000100:00000001:1.0:1788879799.802436:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305411625 00000100:00000001:1.0:1788879799.802438:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879799.802439:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.802440:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879799.802443:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879799/real 1788879799] req@ffff94e5045e4e00 x1875765290199808/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879799.802447:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879799.802448:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879799.802449:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879799.802451:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879799.802452:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802453:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879799.802454:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879799.802456:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290199808/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879799.802460:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802461:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879799.802463:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802464:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879799.802464:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802465:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879799.802478:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879799.802479:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879799.802482:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879799.802483:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879799.802485:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290199808:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879799.802488:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802489:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879799.802490:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879799.802490:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879799.802491:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879799.802492:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879799.802493:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879799.802494:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879799.802495:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879799.802496:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879799.802497:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802498:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.802500:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879799.802501:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.802502:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879799.802503:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879799.865044:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879799.865053:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879799.865056:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879799.865063:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879799.865068:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879799.865073:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.865076:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879799.865080:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879799.865084:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879800.569024:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879800.569031:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879800.569036:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879800.569037:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879800.569108:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879800.569113:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879800.569118:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879800.889003:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879800.889011:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879800.889012:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879800.889017:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879800.889020:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879800.889024:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879800.889027:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879800.889029:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879800.889031:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879801.593016:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879801.593026:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879801.593033:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879801.593034:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879801.593076:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879801.593079:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879801.593081:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879801.912985:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879801.912993:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879801.912994:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879801.913015:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879801.913019:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879801.913022:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879801.913025:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879801.913028:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879801.913031:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879802.617008:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879802.617015:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879802.617019:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879802.617020:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879802.617059:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879802.617062:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879802.617064:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879802.936958:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879802.936965:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879802.936981:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879802.936987:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879802.936990:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879802.936994:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879802.936996:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879802.936998:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879802.937000:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879803.641018:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879803.641026:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879803.641031:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879803.641032:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879803.641125:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879803.641131:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879803.641136:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879803.960990:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879803.960997:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879803.960998:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879803.961003:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879803.961006:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879803.961009:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879803.961011:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879803.961013:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879803.961016:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879804.665069:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879804.665080:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879804.665098:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879804.665100:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879804.665249:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879804.665255:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879804.665261:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879804.921155:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879804.921180:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879804.921185:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879804.921191:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879804.921200:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879804.921212:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10743 00000100:00080000:3.0:1788879804.921229:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879804.921234:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879804.921238:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879804.921251:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879804.921255:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879804.921260:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879804.921264:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879804.921268:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879804.921270:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879804.921276:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879804.921279:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879804.921283:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879804.921313:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879804.921318:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.921322:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879804.921336:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879804.921352:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879804.921355:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.921360:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879804.921364:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879804.921371:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.921390:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879804.921406:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290199936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879804.921422:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879804.921429:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.921450:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879804.921457:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879804.921463:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879804.921467:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879804.921471:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879804.921476:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879804.921482:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10743 00000100:00000001:1.0:1788879804.921489:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1788879804.921489:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879804.921496:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879804.921501:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879804.921504:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1788879804.921507:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879804.921513:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1788879804.921513:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879804.921520:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879804.921524:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1788879804.921526:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879804.921532:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.921533:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879804.921538:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879804.921545:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879804.921548:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879804.921550:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879804.921559:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879804.921563:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879804.921568:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879804.921570:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.921579:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879804.921583:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879804.921596:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290199936:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879804.921604:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879804.921622:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879804.921627:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879804.921633:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879804.921640:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.921669:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879804.921673:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:1.0:1788879804.921692:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:3.0:1788879804.921716:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879804.921722:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.921726:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879804.921734:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879804.921738:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.921740:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879804.921742:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000800:00000100:1.0:1788879804.921796:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879804.921816:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879804.921822:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879804.921825:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.921828:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879804.921836:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.921843:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305416745 00000100:00000001:1.0:1788879804.921870:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788879804.921879:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305416745 00000100:00000001:1.0:1788879804.921886:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879804.921891:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879804.921895:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.921899:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879804.921986:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879804/real 1788879804] req@ffff94e50449df80 x1875765290199936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879804.921988:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879804.922014:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879804.922018:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879804.922040:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879804.922047:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879804.922052:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.922057:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879804.922062:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879804.922072:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290199936/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879804.922113:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879804.922113:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.922126:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:3.0:1788879804.922133:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e680 x1875765290200064/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879804.922136:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.922147:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879804.922153:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.922165:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879804.922166:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879804.922176:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.922183:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879804.922185:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879804.922191:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1788879804.922193:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879804.922203:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1788879804.922206:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:3.0:1788879804.922208:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879804.922215:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879804.922217:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1788879804.922223:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00100000:1.0:1788879804.922229:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290199936:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1788879804.922235:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10743 00000100:00000001:1.0:1788879804.922247:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1788879804.922248:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879804.922258:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879804.922259:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879804.922267:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879804.922268:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879804.922276:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:3.0:1788879804.922277:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879804.922284:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879804.922308:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 02000000:00000001:1.0:1788879804.922309:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1788879804.922313:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:1.0:1788879804.922316:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000100:00000001:3.0:1788879804.922321:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000020:00000001:1.0:1788879804.922323:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 00000100:00000001:3.0:1788879804.922326:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879804.922333:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:3.0:1788879804.922335:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 02000000:00000001:1.0:1788879804.922340:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1788879804.922343:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879804.922348:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879804.922349:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 02000000:00000001:3.0:1788879804.922353:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879804.922355:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879804.922366:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.922377:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879804.922378:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1788879804.922384:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879804.922390:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.922392:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.922395:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1788879804.922400:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879804.922402:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879804.922405:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.922413:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879804.922417:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788879804.922418:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879804.922425:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.922428:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879804.922434:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.922437:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879804.922445:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1788879804.922446:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449df80 x1875765290200192/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879804.922453:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879804.922466:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879804.922469:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1788879804.922475:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879804.922475:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.922484:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879804.922488:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879804.922490:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879804.922501:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00100000:1.0:1788879804.922505:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290200064:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879804.922506:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879804.922513:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879804.922520:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879804.922523:0:178134:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:1.0:1788879804.922526:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879804.922533:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879804.922534:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10743 02000000:00000001:1.0:1788879804.922542:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788879804.922555:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788879804.922560:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.922562:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879804.922567:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879804.922574:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879804.922576:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879804.922579:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1788879804.922580:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879804.922587:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073214592 : -117763636337024 : ffff94e50449e680) 00000020:00000001:3.0:1788879804.922588:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879804.922598:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879804.922600:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879804.922601:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879804.922608:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879804.922613:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879804.922617:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879804.922621:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879804.922625:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.922628:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879804.922632:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1788879804.922635:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000008:00000001:3.0:1788879804.922638:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.922644:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.922648:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:3.0:1788879804.922653:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879804.922657:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1788879804.922658:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.922665:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.922666:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879804.922671:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879804.922674:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:3.0:1788879804.922677:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.922682:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.922688:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.922692:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305416204 00000100:00100000:3.0:1788879804.922702:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ce00 x1875765290200320/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879804.922703:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305416745 00000100:00000001:1.0:1788879804.922713:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305416204 00000100:00000001:1.0:1788879804.922734:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305416745 00000100:00000001:3.0:1788879804.922739:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879804.922742:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879804.922747:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.922750:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879804.922755:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.922757:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879804.922771:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879804/real 1788879804] req@ffff94e50449e680 x1875765290200064/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879804.922783:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879804.922786:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879804.922792:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879804.922795:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879804.922797:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.922800:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879804.922802:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879804.922807:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e680 x1875765290200064/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879804.922826:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.922829:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879804.922832:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.922834:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879804.922835:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.922838:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879804.922854:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879804.922858:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879804.922862:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879804.922864:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879804.922868:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290200064:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879804.922873:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.922876:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879804.922878:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879804.922880:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879804.922883:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879804.922885:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879804.922886:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879804.922889:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879804.922891:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879804.922894:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879804.922896:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.922898:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.922947:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879804.922951:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.922957:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879804.922972:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879804.922976:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879804.922980:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879804.922984:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879804.922990:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879804.922994:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879804.922998:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879804.923006:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290200192:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879804.923011:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879804.923014:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879804.923017:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879804.923019:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879804.923024:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.923044:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879804.923046:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073212800 : -117763636338816 : ffff94e50449df80) 00000100:00000001:1.0:1788879804.923051:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879804.923070:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879804.923076:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879804.923080:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879804.923082:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.923084:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879804.923087:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.923092:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305416203 00000100:00000001:1.0:1788879804.923095:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305416745 00000100:00000001:1.0:1788879804.923098:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305416203 00000100:00000001:1.0:1788879804.923101:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305416745 00000100:00000001:1.0:1788879804.923104:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879804.923107:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.923109:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879804.923114:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879804/real 1788879804] req@ffff94e50449df80 x1875765290200192/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879804.923124:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879804.923125:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879804.923128:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879804.923130:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879804.923132:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.923134:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879804.923148:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879804.923153:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449df80 x1875765290200192/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879804.923170:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.923173:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879804.923176:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.923178:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879804.923180:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.923182:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879804.923196:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879804.923199:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879804.923203:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879804.923205:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879804.923209:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449df80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290200192:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879804.923214:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879804.923216:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879804.923218:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879804.923221:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879804.923223:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879804.923226:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879804.923229:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290200320:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879804.923233:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879804.923235:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879804.923236:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879804.923239:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879804.923243:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.923248:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879804.923250:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073208320 : -117763636343296 : ffff94e50449ce00) 00000100:00000001:1.0:1788879804.923254:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879804.923265:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879804.923270:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879804.923273:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879804.923274:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.923277:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879804.923280:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.923284:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305416203 00000100:00000001:1.0:1788879804.923313:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305416745 00000100:00000001:1.0:1788879804.923318:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305416203 00000100:00000001:1.0:1788879804.923321:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305416745 00000100:00000001:1.0:1788879804.923324:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879804.923327:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.923329:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879804.923334:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879804/real 1788879804] req@ffff94e50449ce00 x1875765290200320/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879804.923345:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879804.923346:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879804.923350:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879804.923353:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879804.923355:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.923357:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879804.923359:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879804.923364:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ce00 x1875765290200320/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879804.923372:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.923375:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879804.923377:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.923379:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879804.923381:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.923383:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879804.923397:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879804.923400:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879804.923404:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879804.923406:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879804.923411:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290200320:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879804.923416:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.923419:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879804.923421:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879804.923423:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879804.923426:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879804.923427:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879804.923429:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879804.923432:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879804.923434:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879804.923436:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879804.923438:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.923440:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879804.923442:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879804.923443:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879804.923446:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879804.923448:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879804.923449:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879804.923451:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879804.923452:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879804.923455:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879804.923456:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.923458:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.923462:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879804.923464:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.923466:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879804.923468:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879804.985090:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879804.985101:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879804.985103:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879804.985110:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879804.985115:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879804.985119:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.985123:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879804.985143:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879804.985149:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879805.689025:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879805.689033:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879805.689041:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879805.689045:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879805.689048:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879805.689053:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879805.689057:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879805.689059:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879805.689062:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879805.689065:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879805.689093:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879805.689096:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879805.689115:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879806.009055:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879806.009067:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879806.009070:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879806.009079:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879806.009085:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879806.009090:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879806.009094:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879806.009098:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879806.009103:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879806.713025:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879806.713034:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879806.713042:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879806.713043:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879806.713068:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879806.713072:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879806.713076:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879807.033101:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879807.033116:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879807.033121:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879807.033133:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879807.033140:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879807.033146:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879807.033152:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879807.033156:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879807.033161:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879807.737112:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879807.737123:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879807.737133:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879807.737135:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879807.737336:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879807.737345:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879807.737352:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879808.057142:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879808.057157:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879808.057162:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879808.057172:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879808.057180:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879808.057187:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879808.057192:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879808.057198:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879808.057204:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879808.761039:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879808.761047:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879808.761050:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879808.761054:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879808.761060:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879808.761062:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879808.761065:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879808.761067:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879808.761071:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879808.761073:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879808.761203:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879808.761211:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879808.761218:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879809.081086:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879809.081109:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879809.081112:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879809.081120:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879809.081126:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879809.081132:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879809.081136:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879809.081140:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879809.081144:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879809.785055:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879809.785061:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879809.785067:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879809.785068:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879809.785123:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879809.785126:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879809.785129:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879810.041073:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879810.041090:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879810.041095:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879810.041102:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879810.041112:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879810.041126:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10748 00000100:00080000:3.0:1788879810.041147:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879810.041164:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879810.041168:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879810.041185:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879810.041190:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879810.041197:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879810.041202:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879810.041206:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879810.041208:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879810.041217:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879810.041222:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879810.041228:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879810.041235:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879810.041241:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879810.041245:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879810.041250:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879810.041294:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879810.041298:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879810.041304:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879810.041309:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879810.041318:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879810.041340:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879810.041356:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d180 x1875765290200448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879810.041372:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879810.041379:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879810.041404:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879810.041409:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879810.041414:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879810.041416:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879810.041418:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879810.041422:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879810.041426:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10748 00000100:00080000:3.0:1788879810.041431:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879810.041434:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879810.041436:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879810.041441:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879810.041443:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879810.041447:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879810.041450:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879810.041453:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879810.041456:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879810.041463:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879810.041467:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879810.041470:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879810.041478:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879810.041481:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879810.041484:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879810.041488:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:3.0:1788879810.041491:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879810.041499:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.041500:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879810.041502:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.041510:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788879810.041512:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879810.041518:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.041519:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879810.041521:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879810.041528:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879810.041529:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879810.041532:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879810.041545:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879810.041546:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879810.041555:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788879810.041558:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449fb80 x1875765290200576/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879810.041567:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879810.041582:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1788879810.041584:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290200448:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879810.041590:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.041599:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879810.041601:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879810.041618:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1788879810.041619:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879810.041625:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879810.041627:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:1.0:1788879810.041634:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879810.041635:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879810.041640:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879810.041644:0:178134:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:1.0:1788879810.041646:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879810.041652:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10748 00000100:00080000:3.0:1788879810.041658:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879810.041661:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879810.041664:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879810.041668:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879810.041670:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879810.041673:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879810.041677:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1788879810.041678:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879810.041684:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073209216 : -117763636342400 : ffff94e50449d180) 00000100:00000001:3.0:1788879810.041686:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879810.041693:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879810.041695:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879810.041702:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879810.041707:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879810.041709:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879810.041713:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879810.041716:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879810.041719:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879810.041722:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879810.041726:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879810.041728:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879810.041734:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879810.041735:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879810.041738:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879810.041740:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879810.041743:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879810.041750:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879810.041757:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449d880 x1875765290200704/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1788879810.041765:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879810.041780:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879810.041784:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879810.041788:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.041791:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:3.0:1788879810.041793:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879810.041800:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879810.041801:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879810.041806:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1788879810.041808:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879810.041812:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879810.041814:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879810.041823:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879810.041827:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879810.041833:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10748 00000100:00000001:1.0:1788879810.041838:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305421865 00000100:00080000:3.0:1788879810.041841:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879810.041848:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879810.041850:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879810.041854:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879810.041856:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879810.041858:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1788879810.041860:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879810.041867:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879810.041871:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879810.041873:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305421865 00000100:00000001:3.0:1788879810.041874:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879810.041885:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1788879810.041886:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879810.041894:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879810.041895:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879810.041899:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879810.041902:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879810.041942:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879810.041945:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879810.041948:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879810.041951:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879810.041954:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879810.041960:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879810.041962:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879810.041967:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879810.041971:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879810.041972:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879810.041978:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.041990:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879810.041992:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879810.042003:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879810/real 1788879810] req@ffff94e50449d180 x1875765290200448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879810.042006:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c380 x1875765290200832/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879810.042049:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000400:00000001:1.0:1788879810.042050:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788879810.042055:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879810.042057:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879810.042062:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.042066:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879810.042073:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879810.042077:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042082:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879810.042087:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879810.042099:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d180 x1875765290200448/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879810.042116:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042123:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879810.042131:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042134:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879810.042136:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042142:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879810.042145:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879810.042152:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879810.042160:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879810.042174:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879810.042182:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290200448:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879810.042209:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042214:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879810.042217:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879810.042221:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879810.042228:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879810.042232:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879810.042236:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879810.042243:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879810.042276:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879810.042283:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879810.042286:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042291:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042303:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879810.042307:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.042311:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879810.042314:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879810.042317:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879810.042320:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879810.042323:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879810.042328:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879810.042332:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879810.042336:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879810.042344:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290200576:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879810.042351:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879810.042354:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879810.042356:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879810.042359:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879810.042363:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.042374:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879810.042376:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219968 : -117763636331648 : ffff94e50449fb80) 00000100:00000001:1.0:1788879810.042382:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879810.042402:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879810.042409:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879810.042413:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879810.042415:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.042417:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879810.042420:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.042426:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305421324 00000100:00000001:1.0:1788879810.042430:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305421865 00000100:00000001:1.0:1788879810.042433:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305421324 00000100:00000001:1.0:1788879810.042436:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305421865 00000100:00000001:1.0:1788879810.042439:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879810.042441:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.042444:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879810.042450:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879810/real 1788879810] req@ffff94e50449fb80 x1875765290200576/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879810.042460:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879810.042462:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879810.042466:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879810.042469:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879810.042473:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042477:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879810.042480:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879810.042488:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449fb80 x1875765290200576/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879810.042503:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042508:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879810.042513:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042516:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879810.042519:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042523:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879810.042546:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879810.042552:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879810.042559:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879810.042564:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879810.042571:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449fb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290200576:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879810.042579:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879810.042581:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879810.042583:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879810.042586:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879810.042589:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879810.042591:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879810.042595:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290200704:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879810.042599:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879810.042602:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879810.042604:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879810.042607:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879810.042622:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.042631:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879810.042633:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073211008 : -117763636340608 : ffff94e50449d880) 00000100:00000001:1.0:1788879810.042638:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879810.042656:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879810.042662:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879810.042666:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879810.042668:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.042670:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879810.042674:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.042678:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305421322 00000100:00000001:1.0:1788879810.042682:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305421865 00000100:00000001:1.0:1788879810.042685:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305421322 00000100:00000001:1.0:1788879810.042688:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305421865 00000100:00000001:1.0:1788879810.042690:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879810.042693:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.042695:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879810.042700:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879810/real 1788879810] req@ffff94e50449d880 x1875765290200704/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879810.042709:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879810.042711:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879810.042713:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879810.042716:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879810.042718:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042720:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879810.042722:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879810.042726:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449d880 x1875765290200704/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879810.042735:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042737:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879810.042740:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042742:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879810.042743:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042745:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879810.042757:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879810.042760:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879810.042763:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879810.042767:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879810.042771:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449d880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290200704:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879810.042775:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879810.042777:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879810.042779:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879810.042781:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879810.042783:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879810.042786:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879810.042789:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290200832:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879810.042792:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879810.042794:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879810.042796:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879810.042798:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879810.042803:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.042808:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879810.042809:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073205632 : -117763636345984 : ffff94e50449c380) 00000100:00000001:1.0:1788879810.042813:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879810.042824:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879810.042829:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879810.042831:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879810.042833:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.042835:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879810.042838:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.042842:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305421322 00000100:00000001:1.0:1788879810.042845:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305421865 00000100:00000001:1.0:1788879810.042848:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305421322 00000100:00000001:1.0:1788879810.042851:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305421865 00000100:00000001:1.0:1788879810.042854:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879810.042856:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.042858:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879810.042862:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879810/real 1788879810] req@ffff94e50449c380 x1875765290200832/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879810.042881:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879810.042883:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879810.042885:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879810.042887:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879810.042889:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042891:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879810.042892:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879810.042897:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c380 x1875765290200832/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879810.042948:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042951:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879810.042954:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042956:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879810.042957:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042960:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879810.042973:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879810.042976:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879810.042980:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879810.042983:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879810.042987:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290200832:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879810.042992:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.042994:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879810.042996:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879810.042998:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879810.043002:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879810.043004:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879810.043006:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879810.043009:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879810.043020:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879810.043024:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879810.043025:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.043028:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879810.043030:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879810.043031:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879810.043047:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879810.043049:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879810.043051:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879810.043053:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879810.043055:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879810.043062:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879810.043063:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.043066:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879810.043067:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879810.043069:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879810.043071:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879810.043072:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879810.043074:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879810.043076:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879810.043077:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879810.043079:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879810.043081:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.043083:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.043087:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879810.043089:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.043091:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879810.043094:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879810.105061:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879810.105070:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879810.105073:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879810.105081:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879810.105087:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879810.105092:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.105097:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879810.105101:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879810.105106:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879810.809044:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879810.809052:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879810.809060:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879810.809063:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879810.809139:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879810.809142:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879810.809146:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879811.129029:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879811.129041:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879811.129044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879811.129052:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879811.129057:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879811.129063:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879811.129067:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879811.129072:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879811.129076:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879811.833060:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879811.833069:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879811.833076:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879811.833078:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879811.833214:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879811.833224:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879811.833231:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879812.153050:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879812.153061:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879812.153064:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879812.153072:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879812.153077:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879812.153083:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879812.153087:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879812.153091:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879812.153095:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879812.856996:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879812.857004:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879812.857011:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879812.857013:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879812.857163:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879812.857168:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879812.857173:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879813.177015:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879813.177025:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879813.177027:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879813.177035:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879813.177040:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879813.177045:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879813.177048:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879813.177052:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879813.177056:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879813.880996:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879813.881005:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879813.881013:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879813.881014:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879813.881037:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879813.881039:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879813.881041:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879814.201050:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879814.201062:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879814.201065:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879814.201073:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879814.201078:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879814.201083:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879814.201087:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879814.201091:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879814.201095:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879814.905028:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879814.905038:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879814.905048:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879814.905051:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879814.905100:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879814.905105:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879814.905110:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879815.161011:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879815.161024:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879815.161028:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879815.161033:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879815.161041:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879815.161050:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10754 00000100:00080000:3.0:1788879815.161063:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879815.161067:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879815.161070:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879815.161081:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879815.161084:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879815.161089:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879815.161092:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879815.161095:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879815.161097:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879815.161102:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879815.161105:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879815.161109:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879815.161114:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879815.161118:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879815.161121:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879815.161124:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879815.161139:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879815.161141:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879815.161145:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879815.161149:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879815.161156:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879815.161170:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879815.161182:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290200960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879815.161196:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879815.161202:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879815.161246:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879815.161251:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879815.161256:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879815.161258:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879815.161261:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879815.161265:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879815.161269:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10754 00000100:00080000:3.0:1788879815.161274:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879815.161277:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879815.161279:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879815.161284:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879815.161286:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879815.161289:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879815.161293:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879815.161296:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879815.161298:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879815.161303:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879815.161307:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879815.161311:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879815.161328:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879815.161332:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879815.161334:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879815.161339:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879815.161343:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.161347:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879815.161347:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879815.161354:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879815.161357:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.161358:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1788879815.161364:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879815.161367:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1788879815.161370:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879815.161374:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788879815.161375:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.161379:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879815.161385:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879815.161390:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879815.161396:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788879815.161398:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290201088/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879815.161406:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879815.161433:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290200960:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879815.161434:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879815.161447:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.161466:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879815.161472:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879815.161475:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879815.161480:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879815.161486:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1788879815.161487:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879815.161499:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:1.0:1788879815.161500:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879815.161513:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879815.161523:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879815.161530:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879815.161535:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879815.161538:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00080000:3.0:1788879815.161541:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10754 00000100:00000001:1.0:1788879815.161546:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1788879815.161559:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879815.161565:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879815.161568:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879815.161576:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879815.161579:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879815.161582:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879815.161585:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879815.161589:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879815.161592:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879815.161598:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879815.161602:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879815.161606:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879815.161612:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879815.161615:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879815.161618:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879815.161621:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879815.161627:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.161634:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305426985 00000008:00000001:3.0:1788879815.161643:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879815.161646:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.161650:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1788879815.161654:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879815.161659:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879815.161662:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879815.161664:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879815.161668:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.161670:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305426985 00000100:00000001:3.0:1788879815.161680:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.161681:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1788879815.161697:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788879815.161700:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290201216/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879815.161703:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.161718:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.161732:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879815.161757:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1788879815.161758:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879815.161769:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1788879815.161772:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.161780:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879815.161785:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879815.161787:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879815.161789:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1788879815.161792:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.161793:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879815.161815:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1788879815.161816:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1788879815.161824:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1788879815.161826:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 02000000:00000001:1.0:1788879815.161829:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879815.161831:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879815.161837:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788879815.161840:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290201088:192.168.203.129@tcp:38:kworker.0 00000100:00000001:3.0:1788879815.161845:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879815.161854:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10754 00000100:00000001:1.0:1788879815.161856:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879815.161864:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1788879815.161864:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788879815.161869:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879815.161873:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879815.161876:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879815.161877:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 02000000:00000001:1.0:1788879815.161885:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879815.161885:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879815.161891:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879815.161895:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879815.161896:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1788879815.161948:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879815.161953:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879815.161957:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879815.161961:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00080000:3.0:1788879815.161968:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879815.161973:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879815.161975:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879815.161981:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879815.161986:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879815.161991:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879815.161994:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879815.161997:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788879815.161998:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879815.162005:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.162006:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305426444 00000100:00000001:3.0:1788879815.162012:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.162015:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305426985 02000000:00000001:3.0:1788879815.162024:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879815.162026:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305426444 02000000:00000001:3.0:1788879815.162035:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.162036:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305426985 00000100:00000001:3.0:1788879815.162047:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879815.162048:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879815.162054:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879815.162055:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879815.162062:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.162063:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879815.162068:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879815.162072:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879815.162078:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879815.162078:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879815.162089:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1788879815.162100:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290201344/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1788879815.162102:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879815.162120:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290201216:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879815.162137:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879815.162143:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879815.162147:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879815.162155:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879815.162161:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879815.162166:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879815.162170:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879815.162175:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:3.0:1788879815.162177:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.162183:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879815.162191:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.162205:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.162214:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305426442 00000100:00000001:1.0:1788879815.162233:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305426985 00000100:00000001:1.0:1788879815.162237:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305426442 00000100:00000001:1.0:1788879815.162240:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305426985 00000100:00000001:1.0:1788879815.162244:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879815.162248:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.162253:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.162257:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879815.162264:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879815.162267:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788879815.162272:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879815.162274:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879815.162277:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879815.162279:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879815.162281:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879815.162284:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879815.162286:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879815.162288:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879815.162292:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290201344:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879815.162296:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879815.162299:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879815.162300:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879815.162302:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879815.162306:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.162312:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879815.162314:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:1.0:1788879815.162320:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1788879815.162335:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.162341:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305426442 00000100:00000001:1.0:1788879815.162345:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305426985 00000100:00000001:1.0:1788879815.162348:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305426442 00000100:00000001:1.0:1788879815.162351:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305426985 00000100:00000001:1.0:1788879815.162354:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879815.162357:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.162360:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.162363:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879815.162367:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879815.162370:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1788879815.162373:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879815.162375:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879815.162377:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.162380:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879815.225106:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879815.225119:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879815.225122:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879815.225129:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879815.225134:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879815.225139:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879815.225142:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879815.225146:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879815.225149:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879815.929028:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879815.929036:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879815.929043:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879815.929048:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879815.929051:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879815.929056:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879815.929060:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879815.929063:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879815.929066:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879815.929069:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879815.929262:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879815.929271:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879815.929278:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879816.249061:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879816.249070:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879816.249073:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879816.249079:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879816.249084:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879816.249090:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879816.249093:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879816.249096:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879816.249100:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879816.953027:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879816.953034:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879816.953040:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879816.953042:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879816.953126:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879816.953129:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879816.953133:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879817.272979:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879817.272987:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879817.272991:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879817.272996:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879817.272999:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879817.273005:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879817.273008:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879817.273010:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879817.273012:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879817.977009:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879817.977015:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879817.977019:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879817.977020:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879817.977046:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879817.977048:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879817.977051:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1788879818.233261:0:2349:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.129:988 00000400:00000100:0.0:1788879818.233305:0:2349:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.129@tcp at host 192.168.203.129 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1788879818.233325:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:0.0:1788879818.233342:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879818.233362:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879818.233367:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879818.233371:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879818.233386:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00000001:1.0:1788879818.233396:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1788879818.233397:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879818.233403:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1788879818.233404:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879818.233408:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879818.233409:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1788879818.233411:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000800:00000100:0.0:1788879818.233422:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000100:00100000:1.0:1788879818.233424:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879815/real 1788879818] req@ffff94e50449f480 x1875765290201088/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1788879818.233431:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:0.0:1788879818.233437:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879818.233442:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879818.233449:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000400:00000001:1.0:1788879818.233460:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000800:00000100:0.0:1788879818.233461:0:2349:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.129@tcp 00000400:00000001:1.0:1788879818.233464:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879818.233474:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879818.233477:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879818.233478:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233482:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1788879818.233485:0:2349:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879818.233486:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1788879818.233490:0:2349:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1788879818.233494:0:2349:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1788879818.233499:0:2349:0:(events.c:69:request_out_callback()) Process leaving 00000100:00100000:1.0:1788879818.233499:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290201088/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879818.233515:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233533:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879818.233538:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233541:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879818.233542:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233546:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879818.233567:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879818.233573:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879818.233580:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879818.233584:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879818.233590:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290201088:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879818.233597:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879818.233604:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879815/real 1788879818] req@ffff94e50449c700 x1875765290201216/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879818.233618:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879818.233619:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879818.233625:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879818.233627:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879818.233629:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233632:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879818.233635:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879818.233641:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290201216/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879818.233651:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233654:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879818.233657:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233659:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879818.233660:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233663:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879818.233692:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879818.233696:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879818.233702:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879818.233705:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879818.233710:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290201216:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879818.233716:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879818.233722:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879815/real 1788879818] req@ffff94e50449f800 x1875765290200960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879818.233734:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879818.233736:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879818.233741:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879818.233743:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879818.233745:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233748:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879818.233750:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879818.233755:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290200960/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879818.233765:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233768:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879818.233771:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233774:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879818.233776:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233779:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879818.233782:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879818.233785:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879818.233790:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879818.233793:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879818.233797:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290200960:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879818.233803:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879818.233805:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879818.233807:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879818.233811:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879818.233819:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879818.233822:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879818.233825:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879818.233837:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879818.233841:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879818.233846:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879818.233848:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233850:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879818.233852:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879818.233854:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879818.233857:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879818.233859:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879818.233861:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879818.233864:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879818.233865:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879818.233868:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879818.233870:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233872:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879818.233873:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879818.233875:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879818.233877:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879818.233879:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879818.233881:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879818.233884:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879818.233885:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879818.233887:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879818.233890:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233894:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879818.233897:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879818.233899:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879818.233927:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879818.233936:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879815/real 1788879818] req@ffff94e50449e300 x1875765290201344/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879818.233949:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879818.233951:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879818.233956:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879818.233959:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879818.233961:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233964:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879818.233966:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879818.233971:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290201344/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879818.233982:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233985:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879818.233989:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233991:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879818.233992:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.233995:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879818.234016:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879818.234020:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879818.234024:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879818.234027:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879818.234032:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290201344:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879818.234037:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.234039:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879818.234041:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879818.234043:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879818.234046:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879818.234049:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879818.234058:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879818.234062:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879818.234064:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879818.234067:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879818.234068:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.234071:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.234080:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879818.234083:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879818.234086:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879818.234088:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879818.296989:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879818.296995:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879818.296997:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879818.297004:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879818.297008:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879818.297012:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.297016:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879818.297020:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879818.297023:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879819.001054:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879819.001065:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879819.001068:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879819.001073:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879819.001079:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879819.001083:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879819.001085:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879819.001088:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879819.001093:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879819.001095:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879819.001149:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879819.001152:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879819.001156:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879819.320998:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879819.321010:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879819.321015:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879819.321024:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879819.321031:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879819.321037:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879819.321042:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879819.321047:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879819.321067:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879820.025003:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879820.025010:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879820.025016:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879820.025018:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879820.025126:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879820.025132:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879820.025138:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879820.281048:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879820.281059:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879820.281062:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879820.281066:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879820.281071:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879820.281079:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10759 00000100:00080000:3.0:1788879820.281094:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879820.281100:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879820.281104:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879820.281144:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879820.281148:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879820.281154:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879820.281159:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879820.281163:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879820.281165:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879820.281204:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879820.281226:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879820.281230:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879820.281235:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879820.281249:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879820.281252:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879820.281256:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879820.281268:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879820.281288:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879820.281293:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879820.281297:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879820.281304:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879820.281321:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879820.281336:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290201472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879820.281354:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879820.281361:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879820.281414:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879820.281420:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879820.281425:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879820.281427:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879820.281429:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879820.281434:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879820.281438:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10759 00000100:00080000:3.0:1788879820.281445:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879820.281448:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879820.281451:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879820.281456:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879820.281459:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879820.281462:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879820.281465:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879820.281469:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879820.281470:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1788879820.281472:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879820.281480:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1788879820.281482:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879820.281487:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879820.281490:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.281494:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1788879820.281496:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879820.281500:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879820.281508:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879820.281520:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1788879820.281527:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879820.281529:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1788879820.281532:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 02000000:00000001:1.0:1788879820.281538:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1788879820.281541:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879820.281550:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879820.281552:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290201472:192.168.203.129@tcp:250:kworker.0 00000100:00000001:3.0:1788879820.281556:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.281568:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1788879820.281569:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1788879820.281575:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879820.281579:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879820.281585:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1788879820.281586:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879820.281592:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879820.281594:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879820.281596:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879820.281598:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879820.281613:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.281623:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00100000:3.0:1788879820.281625:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f100 x1875765290201600/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879820.281629:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:1.0:1788879820.281640:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879820.281649:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879820.281653:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879820.281659:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879820.281663:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879820.281668:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879820.281671:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879820.281673:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879820.281679:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879820.281684:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10759 00000100:00080000:3.0:1788879820.281690:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879820.281696:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000800:00000100:1.0:1788879820.281698:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:3.0:1788879820.281701:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879820.281709:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879820.281712:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879820.281716:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879820.281718:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:3.0:1788879820.281721:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879820.281727:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879820.281728:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1788879820.281731:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879820.281735:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879820.281742:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879820.281744:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879820.281752:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.281757:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879820.281757:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879820.281767:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879820.281769:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305432105 02000000:00000001:3.0:1788879820.281775:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879820.281782:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879820.281786:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879820.281788:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000008:00000001:3.0:1788879820.281792:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879820.281798:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.281801:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305432105 02000000:00000001:3.0:1788879820.281807:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879820.281812:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.281813:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879820.281816:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879820.281823:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879820.281823:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879820.281826:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.281830:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.281838:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879820.281840:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879820.281850:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879820/real 1788879820] req@ffff94e50449c000 x1875765290201472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1788879820.281856:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449ca80 x1875765290201728/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1788879820.281895:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1788879820.281896:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879820.281937:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879820.281943:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879820.281948:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:1.0:1788879820.281952:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1788879820.281954:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879820.281960:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879820.281962:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879820.281965:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:3.0:1788879820.281968:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879820.281972:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00080000:3.0:1788879820.281978:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10759 00000100:00000001:1.0:1788879820.281980:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.281988:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1788879820.281989:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879820.281992:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879820.281995:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879820.282000:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00100000:1.0:1788879820.282007:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290201472/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1788879820.282009:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879820.282014:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1788879820.282035:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282040:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879820.282045:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282047:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879820.282049:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282052:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879820.282055:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879820.282059:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879820.282065:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879820.282069:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879820.282075:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290201472:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879820.282081:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282084:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879820.282087:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879820.282090:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879820.282095:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879820.282098:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879820.282100:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879820.282105:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879820.282107:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879820.282111:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879820.282113:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282116:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282125:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879820.282129:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282132:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879820.282134:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879820.282137:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879820.282139:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879820.282141:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879820.282144:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879820.282147:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879820.282150:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879820.282156:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290201600:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879820.282162:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879820.282165:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879820.282168:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879820.282171:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879820.282175:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282211:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879820.282213:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073217280 : -117763636334336 : ffff94e50449f100) 00000100:00000001:1.0:1788879820.282219:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879820.282237:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879820.282244:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000020:00000001:3.0:1788879820.282245:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879820.282248:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1788879820.282252:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:1.0:1788879820.282253:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282259:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879820.282260:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879820.282263:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879820.282265:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879820.282272:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305431564 00000100:00080000:3.0:1788879820.282276:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879820.282279:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305432105 00000100:00000001:3.0:1788879820.282283:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282285:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305431564 02000000:00000001:3.0:1788879820.282289:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879820.282291:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305432105 00000100:00000001:1.0:1788879820.282296:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879820.282297:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879820.282301:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879820.282304:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282306:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879820.282309:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879820.282314:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1788879820.282316:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879820/real 1788879820] req@ffff94e50449f100 x1875765290201600/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1788879820.282320:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879820.282326:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879820.282334:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000400:00000001:1.0:1788879820.282336:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1788879820.282338:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879820.282341:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879820.282343:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879820.282347:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:3.0:1788879820.282348:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879820.282351:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282360:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:3.0:1788879820.282360:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282364:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282369:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:3.0:1788879820.282371:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c000 x1875765290201856/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879820.282372:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879820.282383:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f100 x1875765290201600/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1788879820.282394:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879820.282398:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879820.282407:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282418:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282422:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879820.282425:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282427:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879820.282428:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282431:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879820.282454:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879820.282458:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879820.282463:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879820.282466:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879820.282470:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290201600:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879820.282474:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879820.282476:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879820.282477:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879820.282482:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879820.282484:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879820.282486:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879820.282490:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290201728:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879820.282493:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879820.282495:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879820.282497:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879820.282499:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879820.282502:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282509:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879820.282511:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073207424 : -117763636344192 : ffff94e50449ca80) 00000100:00000001:1.0:1788879820.282516:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879820.282530:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879820.282536:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879820.282539:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879820.282540:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282542:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879820.282545:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282549:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305431562 00000100:00000001:1.0:1788879820.282552:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305432105 00000100:00000001:1.0:1788879820.282555:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305431562 00000100:00000001:1.0:1788879820.282557:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305432105 00000100:00000001:1.0:1788879820.282560:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879820.282562:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282564:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879820.282569:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879820/real 1788879820] req@ffff94e50449ca80 x1875765290201728/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879820.282578:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879820.282580:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879820.282583:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879820.282584:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879820.282602:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282605:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879820.282607:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879820.282613:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449ca80 x1875765290201728/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879820.282624:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282627:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879820.282631:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282633:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879820.282635:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282638:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879820.282672:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879820.282675:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879820.282679:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879820.282681:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879820.282684:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449ca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290201728:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879820.282687:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282689:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879820.282690:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879820.282691:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879820.282694:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879820.282695:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879820.282696:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879820.282698:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879820.282699:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879820.282700:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879820.282701:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282702:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879820.282703:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879820.282704:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879820.282706:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879820.282707:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879820.282707:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879820.282708:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879820.282709:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879820.282710:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879820.282711:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282712:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282715:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879820.282716:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282717:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879820.282718:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879820.282719:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879820.282720:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879820.282721:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879820.282723:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879820.282724:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879820.282725:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879820.282728:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290201856:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879820.282730:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879820.282732:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879820.282733:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879820.282734:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879820.282737:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282741:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879820.282742:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073204736 : -117763636346880 : ffff94e50449c000) 00000100:00000001:1.0:1788879820.282745:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879820.282755:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879820.282758:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879820.282760:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879820.282761:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282762:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879820.282764:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282766:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305431562 00000100:00000001:1.0:1788879820.282768:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305432105 00000100:00000001:1.0:1788879820.282770:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305431562 00000100:00000001:1.0:1788879820.282772:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305432105 00000100:00000001:1.0:1788879820.282773:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879820.282774:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282776:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879820.282778:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879820/real 1788879820] req@ffff94e50449c000 x1875765290201856/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879820.282783:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879820.282784:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879820.282786:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879820.282787:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879820.282788:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282790:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879820.282791:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879820.282793:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c000 x1875765290201856/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879820.282798:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282799:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879820.282801:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282802:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879820.282804:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282805:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879820.282817:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879820.282819:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879820.282823:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879820.282825:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879820.282829:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290201856:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879820.282832:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282835:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879820.282836:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879820.282837:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879820.282839:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879820.282841:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879820.282842:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879820.282844:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879820.282846:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879820.282847:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879820.282849:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282851:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.282854:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879820.282856:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.282858:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879820.282860:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879820.344974:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879820.344979:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879820.344981:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879820.344986:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879820.344988:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879820.344993:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.344996:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879820.344999:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879820.345002:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879821.049052:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879821.049062:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879821.049071:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879821.049074:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879821.049252:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879821.049257:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879821.049262:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879821.369006:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879821.369015:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879821.369017:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879821.369024:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879821.369030:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879821.369035:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879821.369039:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879821.369043:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879821.369046:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879822.073098:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879822.073109:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879822.073119:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879822.073122:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879822.073215:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879822.073221:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879822.073226:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879822.393038:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879822.393051:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879822.393056:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879822.393065:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879822.393072:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879822.393078:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879822.393083:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879822.393088:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879822.393094:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879823.097050:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879823.097061:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879823.097071:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879823.097073:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879823.097109:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879823.097114:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879823.097120:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879823.417041:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879823.417050:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879823.417053:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879823.417060:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879823.417065:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879823.417070:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879823.417074:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879823.417078:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879823.417082:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879824.121049:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879824.121058:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879824.121065:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879824.121067:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879824.121210:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879824.121216:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879824.121221:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879824.441028:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879824.441037:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879824.441040:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879824.441047:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879824.441051:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879824.441056:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879824.441060:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879824.441064:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879824.441067:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879825.145001:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879825.145006:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879825.145010:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879825.145013:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879825.145015:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879825.145017:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879825.145020:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879825.145021:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879825.145023:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879825.145025:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879825.145104:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879825.145108:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879825.145111:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879825.401053:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879825.401061:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879825.401064:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879825.401067:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879825.401070:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879825.401076:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10764 00000100:00080000:2.0:1788879825.401086:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879825.401088:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879825.401090:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879825.401096:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879825.401098:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879825.401100:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879825.401102:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879825.401104:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879825.401105:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879825.401108:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879825.401110:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879825.401112:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879825.401115:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879825.401117:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879825.401119:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879825.401121:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879825.401131:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879825.401133:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879825.401136:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879825.401138:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879825.401142:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879825.401167:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879825.401176:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7b80 x1875765290201984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879825.401184:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879825.401187:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879825.401200:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879825.401203:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879825.401207:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879825.401209:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879825.401211:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879825.401214:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879825.401217:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10764 00000100:00080000:2.0:1788879825.401222:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879825.401224:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879825.401226:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879825.401230:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879825.401231:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879825.401233:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:1.0:1788879825.401236:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1788879825.401236:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879825.401238:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879825.401239:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879825.401240:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879825.401243:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1788879825.401243:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879825.401245:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401246:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879825.401248:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879825.401248:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1788879825.401252:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1788879825.401253:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879825.401255:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879825.401255:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879825.401257:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879825.401263:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1788879825.401265:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1788879825.401266:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879825.401268:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879825.401269:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290201984:192.168.203.129@tcp:250:kworker.0 02000000:00000001:2.0:1788879825.401273:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879825.401274:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879825.401275:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879825.401277:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1788879825.401278:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879825.401278:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879825.401280:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879825.401281:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879825.401283:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879825.401285:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879825.401287:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879825.401290:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4000 x1875765290202112/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879825.401298:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879825.401300:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879825.401302:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401304:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1788879825.401305:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879825.401306:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074563456 : -117763634988160 : ffff94e5045e7b80) 00000100:00080000:2.0:1788879825.401308:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1788879825.401310:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879825.401310:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879825.401312:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879825.401315:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879825.401317:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10764 00000100:00080000:2.0:1788879825.401320:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879825.401322:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879825.401323:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879825.401326:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879825.401327:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879825.401330:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:2.0:1788879825.401332:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:2.0:1788879825.401334:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879825.401335:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879825.401338:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879825.401340:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1788879825.401343:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 02000000:00000001:2.0:1788879825.401343:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879825.401346:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879825.401347:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879825.401349:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879825.401351:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879825.401353:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879825.401355:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401356:0:2361:0:(events.c:35:request_out_callback()) Process entered 02000000:00000001:2.0:1788879825.401359:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879825.401360:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1788879825.401360:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879825.401362:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879825.401363:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879825.401363:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879825.401365:0:2361:0:(events.c:69:request_out_callback()) Process leaving 02000000:00000001:2.0:1788879825.401365:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879825.401369:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401370:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879825.401374:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4e00 x1875765290202240/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879825.401375:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305437225 00000100:00000001:2.0:1788879825.401381:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879825.401383:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401384:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1788879825.401385:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879825.401388:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879825.401391:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:2.0:1788879825.401393:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879825.401394:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879825.401397:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879825.401398:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305437225 00000100:00080000:2.0:1788879825.401400:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10764 00000100:00000001:1.0:1788879825.401402:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1788879825.401403:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879825.401405:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879825.401406:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879825.401409:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401411:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1788879825.401412:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879825.401415:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:1.0:1788879825.401417:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879825/real 1788879825] req@ffff94e5045e7b80 x1875765290201984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1788879825.401417:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879825.401419:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879825.401421:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879825.401424:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879825.401426:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000400:00000001:1.0:1788879825.401428:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879825.401430:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1788879825.401430:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879825.401433:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401435:0:2361:0:(events.c:80:reply_in_callback()) Process entered 02000000:00000001:2.0:1788879825.401435:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1788879825.401438:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 02000000:00000001:2.0:1788879825.401438:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879825.401440:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879825.401440:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879825.401443:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879825.401444:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879825.401446:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1788879825.401446:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1788879825.401448:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879825.401450:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879825.401451:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7b80 x1875765290201984/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1788879825.401454:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879825.401456:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879825.401457:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879825.401459:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1788879825.401460:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879825.401461:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401463:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1788879825.401465:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401467:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401469:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879825.401470:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879825.401470:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6d80 x1875765290202368/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879825.401473:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879825.401475:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879825.401478:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788879825.401478:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879825.401480:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879825.401483:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:2.0:1788879825.401483:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401485:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879825.401489:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290201984:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879825.401494:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401496:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879825.401497:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879825.401500:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879825.401503:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879825.401505:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879825.401508:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879825.401511:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879825.401513:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879825.401517:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879825.401518:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401521:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401528:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879825.401530:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401532:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879825.401533:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879825.401534:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879825.401535:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879825.401536:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879825.401538:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879825.401540:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879825.401541:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879825.401543:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290202112:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879825.401546:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879825.401547:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879825.401548:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879825.401550:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879825.401552:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401557:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879825.401558:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074548224 : -117763635003392 : ffff94e5045e4000) 00000100:00000001:1.0:1788879825.401561:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879825.401571:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879825.401574:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879825.401575:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879825.401576:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401578:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879825.401580:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401583:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305436683 00000100:00000001:1.0:1788879825.401585:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305437225 00000100:00000001:1.0:1788879825.401586:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305436683 00000100:00000001:1.0:1788879825.401588:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305437225 00000100:00000001:1.0:1788879825.401589:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879825.401590:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401592:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879825.401594:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879825/real 1788879825] req@ffff94e5045e4000 x1875765290202112/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879825.401599:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879825.401600:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879825.401602:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879825.401603:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879825.401604:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401605:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879825.401606:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879825.401608:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4000 x1875765290202112/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879825.401613:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401615:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879825.401616:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401617:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879825.401618:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401619:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879825.401632:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879825.401633:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879825.401635:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879825.401636:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879825.401638:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290202112:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879825.401640:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879825.401641:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879825.401642:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879825.401643:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879825.401645:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879825.401646:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879825.401648:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290202240:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879825.401649:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879825.401650:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879825.401651:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879825.401652:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879825.401654:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401657:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879825.401658:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074551808 : -117763634999808 : ffff94e5045e4e00) 00000100:00000001:1.0:1788879825.401660:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879825.401665:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879825.401668:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879825.401669:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879825.401670:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401672:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879825.401673:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401675:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305436681 00000100:00000001:1.0:1788879825.401677:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305437225 00000100:00000001:1.0:1788879825.401678:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305436681 00000100:00000001:1.0:1788879825.401680:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305437225 00000100:00000001:1.0:1788879825.401681:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879825.401682:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401683:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879825.401685:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879825/real 1788879825] req@ffff94e5045e4e00 x1875765290202240/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879825.401690:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879825.401691:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879825.401692:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879825.401693:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879825.401693:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401694:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879825.401695:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879825.401698:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4e00 x1875765290202240/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879825.401702:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401703:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879825.401704:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401706:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879825.401706:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401707:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879825.401719:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879825.401721:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879825.401722:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879825.401724:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879825.401726:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290202240:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879825.401728:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879825.401729:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879825.401730:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879825.401731:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879825.401732:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879825.401734:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879825.401735:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290202368:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879825.401737:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879825.401738:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879825.401739:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879825.401740:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879825.401741:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401743:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879825.401744:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074559872 : -117763634991744 : ffff94e5045e6d80) 00000100:00000001:1.0:1788879825.401746:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879825.401751:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879825.401753:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879825.401754:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879825.401755:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401756:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879825.401758:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401759:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305436681 00000100:00000001:1.0:1788879825.401761:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305437225 00000100:00000001:1.0:1788879825.401762:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305436681 00000100:00000001:1.0:1788879825.401764:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305437225 00000100:00000001:1.0:1788879825.401765:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879825.401766:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401767:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879825.401769:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879825/real 1788879825] req@ffff94e5045e6d80 x1875765290202368/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879825.401778:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879825.401779:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879825.401780:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879825.401781:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879825.401782:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401783:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879825.401784:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879825.401786:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6d80 x1875765290202368/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879825.401790:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401791:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879825.401792:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401793:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879825.401794:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401795:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879825.401806:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879825.401808:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879825.401809:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879825.401810:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879825.401812:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290202368:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879825.401814:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401815:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879825.401816:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879825.401817:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879825.401819:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879825.401820:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879825.401821:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879825.401822:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879825.401823:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879825.401825:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879825.401825:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401827:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879825.401828:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879825.401829:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879825.401830:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879825.401831:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879825.401832:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879825.401833:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879825.401834:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879825.401835:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879825.401836:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401837:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879825.401837:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879825.401838:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879825.401839:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879825.401839:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879825.401840:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879825.401841:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879825.401841:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879825.401843:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879825.401843:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401844:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.401846:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879825.401847:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.401848:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879825.401849:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879825.464937:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879825.464940:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879825.464942:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879825.464945:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879825.464947:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879825.464950:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.464952:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879825.464954:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879825.464955:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879826.169001:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879826.169007:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879826.169013:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879826.169014:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879826.169035:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879826.169038:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879826.169041:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879826.489031:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879826.489039:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879826.489041:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879826.489047:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879826.489051:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879826.489055:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879826.489058:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879826.489061:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879826.489064:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879827.193056:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879827.193066:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879827.193077:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879827.193079:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879827.193115:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879827.193121:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879827.193127:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879827.513000:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879827.513011:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879827.513014:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879827.513020:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879827.513025:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879827.513030:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879827.513034:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879827.513038:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879827.513041:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879828.217018:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879828.217026:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879828.217030:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879828.217034:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879828.217040:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879828.217044:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879828.217046:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879828.217049:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879828.217053:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879828.217056:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879828.217196:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879828.217202:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879828.217207:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879828.537041:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879828.537050:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879828.537052:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879828.537059:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879828.537063:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879828.537068:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879828.537072:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879828.537075:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879828.537078:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879829.241028:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879829.241034:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879829.241040:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879829.241041:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879829.241142:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879829.241149:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879829.241155:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879829.560978:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879829.560985:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879829.560987:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879829.560992:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879829.560995:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879829.561001:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879829.561003:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879829.561006:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879829.561009:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879830.265038:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879830.265046:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879830.265051:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879830.265053:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879830.265075:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879830.265077:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879830.265080:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1788879830.521178:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879830.521191:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1788879830.521195:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879830.521200:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879830.521206:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879830.521216:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10769 00000100:00080000:3.0:1788879830.521231:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879830.521234:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879830.521236:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879830.521248:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879830.521252:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879830.521258:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:3.0:1788879830.521260:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:3.0:1788879830.521263:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879830.521264:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879830.521270:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879830.521272:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879830.521276:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879830.521280:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879830.521283:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879830.521286:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:3.0:1788879830.521289:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879830.521307:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879830.521309:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879830.521313:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879830.521316:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879830.521323:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879830.521341:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879830.521353:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449e300 x1875765290202496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879830.521366:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879830.521372:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879830.521399:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879830.521403:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1788879830.521406:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1788879830.521408:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879830.521410:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879830.521413:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:3.0:1788879830.521416:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10769 00000100:00080000:3.0:1788879830.521419:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1788879830.521421:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879830.521422:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879830.521425:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879830.521427:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879830.521429:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:3.0:1788879830.521432:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:3.0:1788879830.521434:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879830.521435:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879830.521439:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879830.521442:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879830.521445:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879830.521452:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879830.521455:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879830.521457:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879830.521463:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879830.521467:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879830.521469:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879830.521473:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879830.521475:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879830.521480:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879830.521481:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1788879830.521483:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879830.521490:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.521495:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879830.521501:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1788879830.521501:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.521510:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1788879830.521518:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f800 x1875765290202624/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879830.521540:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879830.521562:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1788879830.521565:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879830.521571:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879830.521572:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879830.521583:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879830.521584:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879830.521593:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1788879830.521606:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290202496:192.168.203.129@tcp:250:kworker.0 00000100:00080000:3.0:1788879830.521608:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1788879830.521616:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879830.521622:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1788879830.521633:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1788879830.521635:0:178134:0:(import.c:520:import_select_connection()) Process entered 02000000:00000001:1.0:1788879830.521644:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1788879830.521644:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10769 02000000:00000001:1.0:1788879830.521654:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879830.521656:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1788879830.521664:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1788879830.521666:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1788879830.521671:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:3.0:1788879830.521676:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879830.521677:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879830.521680:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000020:00000001:3.0:1788879830.521682:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:3.0:1788879830.521686:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1788879830.521688:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879830.521695:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:3.0:1788879830.521704:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:3.0:1788879830.521711:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879830.521715:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1788879830.521720:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879830.521721:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1788879830.521725:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.521728:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073213696 : -117763636337920 : ffff94e50449e300) 00000100:00000001:3.0:1788879830.521730:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:3.0:1788879830.521742:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1788879830.521748:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.521750:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1788879830.521751:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879830.521763:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1788879830.521765:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879830.521767:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879830.521769:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1788879830.521773:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879830.521780:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1788879830.521787:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449c700 x1875765290202752/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1788879830.521798:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1788879830.521801:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879830.521805:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879830.521809:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1788879830.521825:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:3.0:1788879830.521828:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:3.0:1788879830.521832:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1788879830.521836:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1788879830.521845:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00000001:1.0:1788879830.521848:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00080000:3.0:1788879830.521857:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10769 00000100:00000001:1.0:1788879830.521864:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1788879830.521868:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879830.521871:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879830.521874:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879830.521877:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:3.0:1788879830.521879:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879830.521889:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1788879830.521891:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1788879830.521901:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1788879830.521905:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:3.0:1788879830.521909:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:3.0:1788879830.521914:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1788879830.521965:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305442346 00000100:00000001:1.0:1788879830.521981:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1788879830.521992:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305442346 00000100:00000001:1.0:1788879830.521997:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1788879830.521998:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879830.522001:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879830.522012:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1788879830.522015:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879830.522019:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1788879830.522030:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879830.522037:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1788879830.522041:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879830/real 1788879830] req@ffff94e50449e300 x1875765290202496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879830.522047:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1788879830.522063:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879830.522068:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879830.522073:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1788879830.522073:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879830.522085:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000008:00100000:3.0:1788879830.522086:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879830.522091:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879830.522096:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1788879830.522099:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.522101:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1788879830.522106:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1788879830.522133:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879830.522135:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1788879830.522138:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879830.522143:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1788879830.522146:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1788879830.522147:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449e300 x1875765290202496/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1788879830.522151:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1788879830.522172:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.522177:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522184:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:3.0:1788879830.522186:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e50449f480 x1875765290202880/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879830.522196:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522209:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879830.522216:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1788879830.522220:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1788879830.522224:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1788879830.522226:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.522230:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1788879830.522233:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879830.522240:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879830.522247:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879830.522253:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879830.522260:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449e300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290202496:192.168.203.129@tcp:250:kworker.0 00000100:00000001:1.0:1788879830.522269:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522272:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879830.522277:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879830.522280:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879830.522288:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879830.522292:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879830.522295:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879830.522301:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879830.522304:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879830.522308:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879830.522310:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522313:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522321:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879830.522324:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.522327:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879830.522329:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879830.522332:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879830.522334:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879830.522336:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879830.522339:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879830.522342:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879830.522345:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879830.522350:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290202624:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879830.522354:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879830.522357:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879830.522359:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879830.522362:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879830.522366:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.522373:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879830.522375:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073219072 : -117763636332544 : ffff94e50449f800) 00000100:00000001:1.0:1788879830.522381:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879830.522398:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879830.522404:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879830.522407:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879830.522408:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.522411:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879830.522414:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.522420:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305441804 00000100:00000001:1.0:1788879830.522423:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305442346 00000100:00000001:1.0:1788879830.522426:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305441804 00000100:00000001:1.0:1788879830.522429:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305442346 00000100:00000001:1.0:1788879830.522432:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879830.522434:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.522436:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879830.522442:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879830/real 1788879830] req@ffff94e50449f800 x1875765290202624/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879830.522452:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879830.522453:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879830.522457:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879830.522459:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879830.522461:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522463:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879830.522465:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879830.522469:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f800 x1875765290202624/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879830.522487:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522490:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879830.522493:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522495:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879830.522496:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522498:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879830.522516:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879830.522519:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879830.522523:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879830.522526:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879830.522530:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290202624:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879830.522535:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879830.522537:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879830.522538:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879830.522541:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879830.522544:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879830.522546:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879830.522549:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290202752:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879830.522553:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879830.522555:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879830.522557:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879830.522559:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879830.522562:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.522568:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879830.522569:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073206528 : -117763636345088 : ffff94e50449c700) 00000100:00000001:1.0:1788879830.522573:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879830.522585:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879830.522589:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879830.522591:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879830.522593:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.522596:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879830.522599:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.522603:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305441802 00000100:00000001:1.0:1788879830.522606:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305442346 00000100:00000001:1.0:1788879830.522609:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305441802 00000100:00000001:1.0:1788879830.522612:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305442346 00000100:00000001:1.0:1788879830.522614:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879830.522616:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.522618:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879830.522623:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879830/real 1788879830] req@ffff94e50449c700 x1875765290202752/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879830.522632:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879830.522633:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879830.522636:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879830.522638:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879830.522639:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522641:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879830.522643:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879830.522647:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449c700 x1875765290202752/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879830.522654:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522657:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879830.522660:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522662:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879830.522663:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522666:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879830.522683:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879830.522686:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879830.522690:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879830.522693:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879830.522697:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449c700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290202752:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879830.522701:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879830.522703:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879830.522705:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879830.522708:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879830.522710:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879830.522712:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879830.522716:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290202880:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879830.522719:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879830.522721:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879830.522733:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879830.522735:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879830.522738:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.522743:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879830.522745:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310073218176 : -117763636333440 : ffff94e50449f480) 00000100:00000001:1.0:1788879830.522748:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879830.522759:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879830.522762:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879830.522765:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879830.522766:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.522768:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879830.522771:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.522775:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305441802 00000100:00000001:1.0:1788879830.522778:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305442346 00000100:00000001:1.0:1788879830.522781:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305441802 00000100:00000001:1.0:1788879830.522784:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305442346 00000100:00000001:1.0:1788879830.522786:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879830.522788:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.522790:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879830.522795:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879830/real 1788879830] req@ffff94e50449f480 x1875765290202880/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879830.522812:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879830.522813:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879830.522816:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879830.522818:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879830.522820:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522821:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879830.522823:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879830.522827:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e50449f480 x1875765290202880/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879830.522835:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522837:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879830.522840:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522842:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879830.522843:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522846:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879830.522868:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879830.522874:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879830.522880:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879830.522884:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879830.522890:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e50449f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290202880:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879830.522937:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522943:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879830.522945:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879830.522948:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879830.522952:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879830.522955:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879830.522960:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879830.522965:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879830.522967:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879830.522971:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879830.522973:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.522978:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879830.522980:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879830.522982:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879830.522986:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879830.522988:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879830.522990:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879830.522993:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879830.522995:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879830.522998:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879830.523000:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.523003:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879830.523005:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879830.523007:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879830.523010:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879830.523013:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879830.523015:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879830.523017:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879830.523019:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879830.523021:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879830.523024:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.523026:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.523032:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879830.523034:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.523038:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879830.523041:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879830.585003:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879830.585010:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879830.585013:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879830.585019:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879830.585023:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879830.585027:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.585030:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879830.585034:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879830.585037:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879831.289082:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879831.289091:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879831.289098:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879831.289100:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879831.289174:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879831.289177:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879831.289181:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879831.609024:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879831.609034:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879831.609037:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879831.609044:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879831.609049:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879831.609054:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879831.609058:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879831.609062:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879831.609066:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879832.313049:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879832.313060:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879832.313066:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879832.313068:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879832.313219:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879832.313231:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879832.313240:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879832.633018:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879832.633027:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879832.633031:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879832.633038:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879832.633042:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879832.633048:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879832.633052:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879832.633056:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879832.633059:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879833.336972:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879833.336982:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879833.337002:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879833.337004:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879833.337082:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879833.337098:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879833.337103:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879833.657038:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879833.657048:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879833.657053:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879833.657059:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879833.657065:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879833.657069:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879833.657072:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879833.657075:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879833.657078:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879834.361105:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879834.361118:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879834.361141:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879834.361145:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879834.361182:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879834.361186:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879834.361190:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879834.680990:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879834.680996:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879834.680998:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879834.681002:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879834.681005:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879834.681009:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879834.681011:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879834.681013:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879834.681015:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879835.386006:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879835.386034:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879835.386041:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879835.386045:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879835.386048:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879835.386053:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879835.386056:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879835.386059:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879835.386061:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879835.386063:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879835.386094:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879835.386096:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879835.386098:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1788879835.641066:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879835.641093:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1788879835.641096:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879835.641101:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879835.641107:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879835.641117:0:178134:0:(import.c:545:import_select_connection()) MGC192.168.203.129@tcp: connect to NID 192.168.203.129@tcp last attempt 10774 00000100:00080000:2.0:1788879835.641143:0:178134:0:(import.c:630:import_select_connection()) MGC192.168.203.129@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879835.641147:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879835.641150:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879835.641160:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879835.641164:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879835.641168:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000020:00000001:2.0:1788879835.641171:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310132611072 : -117763576940544 : ffff94e507d43800) 00000100:00000001:2.0:1788879835.641173:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879835.641175:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879835.641180:0:178134:0:(import.c:666:import_select_connection()) MGC192.168.203.129@tcp: import ffff94e5034df000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879835.641184:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879835.641187:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879835.641191:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879835.641195:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879835.641198:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:2.0:1788879835.641201:0:178134:0:(obd_class.h:893:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879835.641216:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879835.641218:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879835.641222:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1788879835.641226:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879835.641231:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879835.641267:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879835.641281:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e7480 x1875765290203008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879835.641298:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879835.641322:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879835.641346:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879835.641351:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879835.641355:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1788879835.641357:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879835.641359:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879835.641363:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879835.641367:0:178134:0:(import.c:545:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10774 00000100:00080000:2.0:1788879835.641372:0:178134:0:(import.c:630:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879835.641374:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879835.641376:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879835.641380:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879835.641384:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879835.641387:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000020:00000001:2.0:1788879835.641390:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310056697856 : -117763652853760 : ffff94e5034de000) 00000100:00000001:2.0:1788879835.641394:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879835.641397:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879835.641402:0:178134:0:(import.c:666:import_select_connection()) lustre-MDT0000-mdc-ffff94e504943000: import ffff94e5119e7800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879835.641405:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879835.641408:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879835.641419:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879835.641421:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1788879835.641425:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.641431:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1788879835.641432:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879835.641436:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879835.641440:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:2.0:1788879835.641442:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879835.641444:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879835.641457:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:2.0:1788879835.641457:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879835.641469:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879835.641470:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879835.641486:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879835.641491:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1788879835.641498:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879835.641500:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290203008:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879835.641513:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879835.641515:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1788879835.641520:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1788879835.641524:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1788879835.641528:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879835.641529:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879835.641540:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1788879835.641549:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879835.641551:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1788879835.641591:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e4700 x1875765290203136/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879835.641608:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879835.641612:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074561664 : -117763634989952 : ffff94e5045e7480) 00000100:00000001:1.0:1788879835.641624:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1788879835.641633:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879835.641637:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879835.641662:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1788879835.641665:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1788879835.641670:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1788879835.641674:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879835.641676:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1788879835.641682:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879835.641689:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10774 00000800:00000100:1.0:1788879835.641693:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00080000:2.0:1788879835.641710:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1788879835.641719:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:2.0:1788879835.641719:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879835.641725:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879835.641730:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879835.641736:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879835.641737:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1788879835.641740:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000020:00000001:2.0:1788879835.641743:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879835.641749:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879835.641750:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879835.641762:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297573376 : -117763411978240 : ffff94e511a95800) 00000100:00000001:1.0:1788879835.641763:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 0 is 4305447465 00000100:00000001:2.0:1788879835.641783:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879835.641792:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00000001:1.0:1788879835.641801:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1788879835.641804:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0000-osc-ffff94e504943000: import ffff94e5119e5800 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:2.0:1788879835.641812:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879835.641815:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879835.641819:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879835.641822:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879835.641824:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000100:00000001:1.0:1788879835.641825:0:2361:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS new deadline estimation for cpu 1 is 4305447465 00000008:00100000:2.0:1788879835.641828:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879835.641834:0:2361:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.129@tcp: PM QoS request now active for cpu 1 00000008:00000001:2.0:1788879835.641836:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879835.641844:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.641845:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1788879835.641851:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1788879835.641854:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879835.641856:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879835.641862:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879835.641864:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1788879835.641865:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879835.641873:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879835.641882:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879835.641885:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879835/real 1788879835] req@ffff94e5045e7480 x1875765290203008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1788879835.641907:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6680 x1875765290203264/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1788879835.641926:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879835.641932:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1788879835.641993:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879835.641997:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1788879835.642001:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642004:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00080000:2.0:1788879835.642009:0:178134:0:(pinger.c:229:ptlrpc_pinger_process_import()) 84ddc421-1849-4a3d-8835-b34ee88528ec->lustre-OST0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1788879835.642010:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879835.642015:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.642024:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1788879835.642027:0:178134:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0001_UUID: starting recovery 00000100:00000001:1.0:1788879835.642028:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879835.642035:0:178134:0:(import.c:746:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1788879835.642041:0:178134:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1788879835.642044:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e7480 x1875765290203008/t0(0) o250->MGC192.168.203.129@tcp@192.168.203.129@tcp:26/25 lens 520/544 e 0 to 1 dl 1788879860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1788879835.642048:0:178134:0:(import.c:520:import_select_connection()) Process entered 00000100:00080000:2.0:1788879835.642060:0:178134:0:(import.c:545:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: connect to NID 192.168.203.129@tcp last attempt 10774 00000100:00080000:2.0:1788879835.642085:0:178134:0:(import.c:630:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1788879835.642089:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879835.642090:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000020:00000001:2.0:1788879835.642093:0:178134:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1788879835.642098:0:178134:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1788879835.642102:0:178134:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000020:00000001:2.0:1788879835.642107:0:178134:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446626310297567232 : -117763411984384 : ffff94e511a94000) 00000100:00000001:2.0:1788879835.642119:0:178134:0:(connection.c:157:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1788879835.642120:0:178134:0:(connection.c:164:ptlrpc_connection_addref()) Process leaving (rc=18446626310090907968 : -117763618643648 : ffff94e50557e140) 00000100:00080000:2.0:1788879835.642127:0:178134:0:(import.c:666:import_select_connection()) lustre-OST0001-osc-ffff94e504943000: import ffff94e5119e4000 using connection 192.168.203.129@tcp/192.168.203.129@tcp 00000100:00000001:1.0:1788879835.642128:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879835.642133:0:178134:0:(import.c:673:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642138:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1788879835.642138:0:178134:0:(sec.c:1594:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1788879835.642146:0:178134:0:(sec.c:1634:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879835.642149:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1788879835.642154:0:178134:0:(sec.c:1662:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642156:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879835.642161:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1788879835.642161:0:178134:0:(obd_class.h:886:obd_reconnect()) Process entered 00000008:00100000:2.0:1788879835.642168:0:178134:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683431 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1788879835.642170:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879835.642175:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:2.0:1788879835.642181:0:178134:0:(osc_request.c:3833:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1788879835.642186:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5034df000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1788879835.642186:0:178134:0:(obd_class.h:900:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1788879835.642201:0:178134:0:(sec.c:2053:sptlrpc_sepol_get()) Process entered 00000100:00080000:1.0:1788879835.642205:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.129@tcp failed (-110) 02000000:00000001:2.0:1788879835.642206:0:178134:0:(sec.c:2063:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1788879835.642220:0:178134:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1788879835.642222:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1788879835.642225:0:178134:0:(sec.c:439:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1788879835.642230:0:178134:0:(sec.c:463:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879835.642232:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e7480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:a7a063e2-2f26-4c5c-a3bb-1134ca5fa5d6:2361:1875765290203008:192.168.203.129@tcp:250:kworker.0 00000100:00000001:2.0:1788879835.642239:0:178134:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642248:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1788879835.642250:0:178134:0:(import.c:871:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff94e5045e6a00 x1875765290203392/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1788879835.642262:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1788879835.642265:0:178134:0:(jobid.c:959:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1788879835.642269:0:178134:0:(jobid.c:999:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642271:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1788879835.642275:0:178134:0:(import.c:878:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879835.642280:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879835.642292:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879835.642297:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879835.642300:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879835.642309:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879835.642313:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879835.642319:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879835.642322:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.642327:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.642336:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879835.642339:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642342:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879835.642345:0:2361:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879835.642347:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879835.642350:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879835.642352:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879835.642355:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879835.642358:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879835.642361:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879835.642367:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290203136:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879835.642372:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879835.642375:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879835.642377:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879835.642380:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879835.642384:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642393:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879835.642395:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074550016 : -117763635001600 : ffff94e5045e4700) 00000100:00000001:1.0:1788879835.642401:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879835.642421:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879835.642428:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879835.642431:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879835.642433:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642435:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879835.642438:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642443:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305446924 00000100:00000001:1.0:1788879835.642447:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305447465 00000100:00000001:1.0:1788879835.642450:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305446924 00000100:00000001:1.0:1788879835.642452:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305447465 00000100:00000001:1.0:1788879835.642455:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879835.642457:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642459:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879835.642465:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879835/real 1788879835] req@ffff94e5045e4700 x1875765290203136/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879835.642474:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879835.642476:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879835.642479:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879835.642481:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879835.642482:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.642485:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879835.642486:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879835.642491:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e4700 x1875765290203136/t0(0) o38->lustre-MDT0000-mdc-ffff94e504943000@192.168.203.129@tcp:12/10 lens 520/544 e 0 to 1 dl 1788879860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879835.642507:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.642510:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879835.642512:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.642514:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879835.642516:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.642518:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879835.642536:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879835.642539:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e7800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879835.642543:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879835.642545:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879835.642549:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e4700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290203136:192.168.203.129@tcp:38:kworker.0 00000100:00000001:1.0:1788879835.642553:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879835.642555:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879835.642557:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879835.642560:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879835.642563:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879835.642565:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879835.642568:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290203264:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879835.642571:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879835.642574:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879835.642575:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879835.642595:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879835.642598:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642605:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879835.642607:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558080 : -117763634993536 : ffff94e5045e6680) 00000100:00000001:1.0:1788879835.642612:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879835.642625:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879835.642630:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879835.642633:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879835.642634:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642637:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879835.642640:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642644:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305446922 00000100:00000001:1.0:1788879835.642647:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305447465 00000100:00000001:1.0:1788879835.642650:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305446922 00000100:00000001:1.0:1788879835.642652:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305447465 00000100:00000001:1.0:1788879835.642655:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879835.642657:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642659:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879835.642664:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879835/real 1788879835] req@ffff94e5045e6680 x1875765290203264/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879835.642673:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879835.642674:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879835.642677:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879835.642679:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879835.642680:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.642682:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879835.642684:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879835.642688:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6680 x1875765290203264/t0(0) o8->lustre-OST0000-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879835.642696:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.642699:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879835.642702:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.642704:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879835.642706:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.642708:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879835.642724:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879835.642727:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e5800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879835.642730:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879835.642733:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879835.642738:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290203264:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879835.642742:0:2361:0:(client.c:1814:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1788879835.642744:0:2361:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879835.642746:0:2361:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879835.642749:0:2361:0:(sec.c:715:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1788879835.642751:0:2361:0:(sec.c:746:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1788879835.642753:0:2361:0:(sec.c:790:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879835.642756:0:2361:0:(client.c:1920:ptlrpc_send_new_req()) Sending RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290203392:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879835.642760:0:2361:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1788879835.642762:0:2361:0:(sec.c:1086:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1788879835.642764:0:2361:0:(sec.c:1123:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1788879835.642783:0:2361:0:(sec.c:1870:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1788879835.642786:0:2361:0:(sec.c:1880:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642791:0:2361:0:(client.c:3276:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1788879835.642793:0:2361:0:(client.c:3278:ptlrpc_request_addref()) Process leaving (rc=18446626310074558976 : -117763634992640 : ffff94e5045e6a00) 00000100:00000001:1.0:1788879835.642796:0:2361:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1788879835.642807:0:2361:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.129@tcp 00000100:00000001:1.0:1788879835.642811:0:2361:0:(events.c:35:request_out_callback()) Process entered 00000100:00000001:1.0:1788879835.642814:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879835.642816:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642818:0:2361:0:(events.c:69:request_out_callback()) Process leaving 00000100:00000001:1.0:1788879835.642820:0:2361:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642824:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 0 is 4305446922 00000100:00000001:1.0:1788879835.642827:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 0, keep current deadline 4305447465 00000100:00000001:1.0:1788879835.642830:0:2361:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS new deadline estimation for cpu 1 is 4305446922 00000100:00000001:1.0:1788879835.642832:0:2361:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0001-osc-ffff94e504943000: PM QoS request active for cpu 1, keep current deadline 4305447465 00000100:00000001:1.0:1788879835.642835:0:2361:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1788879835.642837:0:2361:0:(client.c:1942:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.642840:0:2361:0:(client.c:2486:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1788879835.642847:0:2361:0:(client.c:2504:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1788879835/real 1788879835] req@ffff94e5045e6a00 x1875765290203392/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1788879835.642900:0:2361:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1788879835.642903:0:2361:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1788879835.642908:0:2361:0:(events.c:80:reply_in_callback()) Process entered 00000100:00000001:1.0:1788879835.642912:0:2361:0:(events.c:165:reply_in_callback()) Process leaving 00000100:00000001:1.0:1788879835.642915:0:2361:0:(client.c:3002:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.642919:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879835.642922:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1788879835.642930:0:2361:0:(client.c:2535:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff94e5045e6a00 x1875765290203392/t0(0) o8->lustre-OST0001-osc-ffff94e504943000@192.168.203.129@tcp:28/4 lens 520/544 e 0 to 1 dl 1788879860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1788879835.642946:0:2361:0:(client.c:2540:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.642951:0:2361:0:(client.c:2125:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1788879835.642955:0:2361:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.642959:0:2361:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1788879835.642961:0:2361:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.642998:0:2361:0:(import.c:1102:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1788879835.643016:0:2361:0:(import.c:1137:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1788879835.643020:0:2361:0:(import.c:65:import_set_state_nolock()) ffff94e5119e4000 lustre-OST0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1788879835.643025:0:2361:0:(import.c:1556:ptlrpc_connect_interpret()) recovery of lustre-OST0001_UUID on 192.168.203.129@tcp failed (-110) 00000100:00000001:1.0:1788879835.643028:0:2361:0:(import.c:1562:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1788879835.643032:0:2361:0:(client.c:2415:ptlrpc_check_set()) Completed RPC req@ffff94e5045e6a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:84ddc421-1849-4a3d-8835-b34ee88528ec:2361:1875765290203392:192.168.203.129@tcp:8:kworker.0 00000100:00000001:1.0:1788879835.643037:0:2361:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.643040:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879835.643043:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879835.643045:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879835.643049:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879835.643051:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879835.643052:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879835.643056:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879835.643057:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879835.643060:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879835.643062:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.643065:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879835.643066:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879835.643068:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879835.643070:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879835.643115:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879835.643116:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879835.643129:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879835.643131:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879835.643133:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879835.643135:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.643137:0:2361:0:(client.c:2868:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1788879835.643139:0:2361:0:(client.c:2794:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1788879835.643140:0:2361:0:(sec.c:1895:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1788879835.643142:0:2361:0:(sec.c:1911:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1788879835.643143:0:2361:0:(genops.c:1216:class_import_put()) Process entered 00000020:00000001:1.0:1788879835.643145:0:2361:0:(genops.c:1229:class_import_put()) Process leaving 02000000:00000001:1.0:1788879835.643146:0:2361:0:(sec.c:479:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1788879835.643148:0:2361:0:(sec.c:496:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1788879835.643149:0:2361:0:(client.c:2850:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1788879835.643151:0:2361:0:(client.c:2909:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.643152:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.643156:0:2361:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879835.643158:0:2361:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.643160:0:2361:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1788879835.643162:0:2361:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1788879835.705035:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879835.705041:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879835.705044:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879835.705049:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879835.705053:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879835.705057:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.705060:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879835.705063:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879835.705067:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879836.408992:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879836.409000:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879836.409007:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879836.409009:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1788879836.409164:0:177866:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1788879836.409169:0:177866:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1788879836.409174:0:177866:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879836.729043:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879836.729052:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879836.729055:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879836.729062:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879836.729090:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879836.729095:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879836.729099:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879836.729103:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879836.729107:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879837.433044:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879837.433051:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879837.433057:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879837.433087:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879837.433167:0:176186:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879837.433174:0:176186:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879837.433180:0:176186:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879837.753042:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879837.753053:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879837.753073:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879837.753081:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879837.753087:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879837.753093:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879837.753098:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879837.753102:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879837.753107:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879838.457093:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879838.457121:0:178237:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:0.0:1788879838.457125:0:178237:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:0.0:1788879838.457131:0:178237:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879838.457140:0:178237:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:0.0:1788879838.457144:0:178237:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879838.457149:0:178237:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879838.457157:0:178237:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879838.457166:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879838.457169:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1788879838.457224:0:156526:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:2.0:1788879838.457231:0:156526:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:2.0:1788879838.457236:0:156526:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1788879838.777004:0:177862:0:(client.c:2568:ptlrpc_expired_set()) Process entered 00000100:00000001:1.0:1788879838.777012:0:177862:0:(client.c:2640:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1788879838.777015:0:177862:0:(client.c:2668:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1788879838.777020:0:177862:0:(client.c:2716:ptlrpc_set_wait()) set ffff94e50af69c00 going to sleep for 0 seconds 00000100:00000001:1.0:1788879838.777024:0:177862:0:(client.c:1991:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1788879838.777029:0:177862:0:(client.c:2994:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879838.777032:0:177862:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1788879838.777036:0:177862:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1788879838.777039:0:177862:0:(client.c:2466:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879839.481039:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879839.481056:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:0.0:1788879839.481062:0:178237:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:0.0:1788879839.481063:0:178237:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1788879839.481085:0:156527:0:(genops.c:1894:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1788879839.481087:0:156527:0:(genops.c:1908:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1788879839.481089:0:156527:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 132815 lines, 132815 kept, 0 dropped, 0 bad.